From e64e1918eeb88e93f9f201ece343624fb2943e9d Mon Sep 17 00:00:00 2001 From: helegraf Date: Fri, 2 Jun 2023 09:26:14 +0200 Subject: [PATCH 01/57] Update stale report to include issues with recent interactions (#1024) Add recently updated issues to list of reminders. --- .github/workflows/stale_reminder.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale_reminder.yml b/.github/workflows/stale_reminder.yml index bed720a61..ce1327683 100644 --- a/.github/workflows/stale_reminder.yml +++ b/.github/workflows/stale_reminder.yml @@ -9,10 +9,23 @@ jobs: stale-reminder: runs-on: ubuntu-latest steps: - - name: Get cutoff date for soon-to-be-stale issues + - name: Get cutoff dates id: date run: | echo "CUTOFF_DATE=$(date -d '-46 days' '+%Y-%m-%d')" >> $GITHUB_ENV + echo "RECENT_ISSUE_CUTOFF_DATE=$(date -d '-7 days' '+%Y-%m-%d')" >> $GITHUB_ENV + - name: Get list of issues that have had interactions in the last week + id: recent + uses: lee-dohm/select-matching-issues@v1 + with: + format: list + path: "recent_issues.md" + token: ${{ github.token }} + query: >- + is:issue + is:open + updated:>=${{ env.RECENT_ISSUE_CUTOFF_DATE }} + sort:updated-asc - name: Collect issues that may become stale id: stale uses: lee-dohm/select-matching-issues@v1 @@ -21,6 +34,7 @@ jobs: path: "potentially_stale_issues.md" token: ${{ github.token }} query: >- + is:issue is:open -label:dependency,documentation,feature,enhancement,bug,test,example,discussion,duplicate,question updated:<${{ env.CUTOFF_DATE }} @@ -33,6 +47,7 @@ jobs: path: "old_issues.md" token: ${{ github.token }} query: >- + is:issue is:open label:dependency,documentation,feature,enhancement,bug,test,example,discussion,duplicate,question updated:<${{ env.CUTOFF_DATE }} @@ -40,7 +55,8 @@ jobs: - name: Combine issues into mail content id: combine run: | - echo $date + echo "## Issues that have had interaction in the last 7 days
" >> mail.html + echo "$(" >> mail.html echo "## Issues that may become stale in <= 14 days
" >> mail.html echo "$(" >> mail.html echo "## Issues that have not had interaction in the last 46 days but will not go stale due to their labels
" >> mail.html From cb1cd2f6436bc1dd5e963d6a4fe3f9efed4fcf35 Mon Sep 17 00:00:00 2001 From: "C. Benjamins" <75323339+benjamc@users.noreply.github.com> Date: Tue, 6 Jun 2023 14:44:42 +0200 Subject: [PATCH 02/57] Raise error if resource limitation and parallelization is requested (#1023) * Add warnings for parallelism doc * Check for parallelization + pynisher Does not work together, throw error. Does not work bc Dask pickles jobs and pynisher runs the target function in a process. * Fix whitespaces --------- Co-authored-by: Carolin Benjamins --- docs/advanced_usage/9_parallelism.rst | 33 +++++++++++++++++++++++++++ smac/facade/abstract_facade.py | 12 ++++++++++ 2 files changed, 45 insertions(+) diff --git a/docs/advanced_usage/9_parallelism.rst b/docs/advanced_usage/9_parallelism.rst index 9912c782f..67e555215 100644 --- a/docs/advanced_usage/9_parallelism.rst +++ b/docs/advanced_usage/9_parallelism.rst @@ -21,6 +21,39 @@ SMAC supports multiple workers natively via Dask. Just specify ``n_workers`` in When using multiple workers, SMAC is not reproducible anymore. +.. warning :: + + You cannot use resource limitation (pynisher, via the `scenario` arguments `trail_walltime_limit` and `trial_memory_limit`). + This is because pynisher works by running your function inside of a subprocess. + Once in the subprocess, the resources will be limited for that process before running your function. + This does not work together with pickling - which is required by dask to schedule jobs on the cluster, even on a local one. + + +.. warning :: + + Start/run SMAC inside ``if __name__ == "__main__"`` in your script otherwise Dask is not able to correctly + spawn jobs and probably this runtime error will be raised: + + .. code-block :: + + RuntimeError: + An attempt has been made to start a new process before the + current process has finished its bootstrapping phase. + + This probably means that you are not using fork to start your + child processes and you have forgotten to use the proper idiom + in the main module: + + if __name__ == '__main__': + freeze_support() + ... + + The "freeze_support()" line can be omitted if the program + is not going to be frozen to produce an executable. + + + + Running on a Cluster -------------------- You can also pass a custom dask client, e.g. to run on a slurm cluster. diff --git a/smac/facade/abstract_facade.py b/smac/facade/abstract_facade.py index adb141112..0aa9dcbb1 100644 --- a/smac/facade/abstract_facade.py +++ b/smac/facade/abstract_facade.py @@ -455,6 +455,18 @@ def _validate(self) -> None: # Make sure the same acquisition function is used assert self._acquisition_function == self._acquisition_maximizer._acquisition_function + if isinstance(self._runner, DaskParallelRunner) and ( + self.scenario.trial_walltime_limit is not None or self.scenario.trial_memory_limit is not None + ): + # This is probably due to pickling dask jobs + raise ValueError( + "Parallelization via Dask cannot be used in combination with limiting " + "the resources " + "of the target function via `scenario.trial_walltime_limit` or " + "`scenario.trial_memory_limit`. Set those to `None` if you want " + "parallelization. " + ) + def _get_signature_arguments(self) -> list[str]: """Returns signature arguments, which are required by the intensifier.""" arguments = [] From 787f5d644acb63c59151b3a9b94696bd737ab739 Mon Sep 17 00:00:00 2001 From: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Date: Tue, 6 Jun 2023 14:45:04 +0200 Subject: [PATCH 03/57] Raise an error for invalid scatter data (#1018) * [enhance] Add an error for scatter data * Update ChangeLog --------- Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> --- CHANGELOG.md | 3 +++ smac/facade/abstract_facade.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56feda166..d8dbab399 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # 2.0.2 +## Improvements +- Add an error when we get an empty dict data_to_scatter so that we can avoid an internal error caused in Dask precautiously + ## Bugfixes - Fix bug in the incumbent selection in the case that multi-fidelity is combined with multi-objective (#1019). diff --git a/smac/facade/abstract_facade.py b/smac/facade/abstract_facade.py index 0aa9dcbb1..1022034c8 100644 --- a/smac/facade/abstract_facade.py +++ b/smac/facade/abstract_facade.py @@ -310,6 +310,9 @@ def optimize(self, *, data_to_scatter: dict[str, Any] | None = None) -> Configur Best found configuration. """ incumbents = None + if isinstance(data_to_scatter, dict) and len(data_to_scatter) == 0: + raise ValueError("data_to_scatter must be None or dict with some elements, but got an empty dict.") + try: incumbents = self._optimizer.optimize(data_to_scatter=data_to_scatter) finally: From b1695f081c7129d14fa01cb19a43c163b765fb4f Mon Sep 17 00:00:00 2001 From: Eddie Bergman Date: Fri, 23 Jun 2023 15:34:48 +0200 Subject: [PATCH 04/57] fix(configspace): shows all keys in print (#1043) --- smac/utils/configspace.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/smac/utils/configspace.py b/smac/utils/configspace.py index e6fdeddbd..8f281ed3e 100644 --- a/smac/utils/configspace.py +++ b/smac/utils/configspace.py @@ -169,12 +169,17 @@ def print_config_changes( if incumbent is None or challenger is None: return - params = sorted([(param, incumbent[param], challenger[param]) for param in challenger.keys()]) - for param in params: - if param[1] != param[2]: - logger.info("--- %s: %r -> %r" % param) - else: - logger.debug("--- %s Remains unchanged: %r", param[0], param[1]) + inc_keys = set(incumbent.keys()) + all_keys = inc_keys.union(challenger.keys()) + + lines = [] + for k in sorted(all_keys): + inc_k = incumbent.get(k, "-inactive-") + cha_k = challenger.get(k, "-inactive-") + lines.append(f"--- {k}: {inc_k} -> {cha_k}" + " (unchanged)" if inc_k == cha_k else "") + + msg = "\n".join(lines) + logger.debug(msg) # def check_subspace_points( From 345d9f1be710996adcfa82493cde4becacaa19aa Mon Sep 17 00:00:00 2001 From: "C. Benjamins" <75323339+benjamc@users.noreply.github.com> Date: Mon, 26 Jun 2023 09:33:43 +0200 Subject: [PATCH 05/57] Fix callback order (#1040) Co-authored-by: Carolin Benjamins --- CHANGELOG.md | 1 + smac/main/smbo.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8dbab399..d7d565b22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ## Bugfixes - Fix bug in the incumbent selection in the case that multi-fidelity is combined with multi-objective (#1019). +- Fix callback order (#1040). # 2.0.1 diff --git a/smac/main/smbo.py b/smac/main/smbo.py index 30e01c2e9..a0ec60caa 100644 --- a/smac/main/smbo.py +++ b/smac/main/smbo.py @@ -461,18 +461,22 @@ def _add_results(self) -> None: logger.info("Cost threshold was reached. Abort is requested.") self._stop = True - def register_callback(self, callback: Callback, index: int = -1) -> None: + def register_callback(self, callback: Callback, index: int | None = None) -> None: """ Registers a callback to be called before, in between, and after the Bayesian optimization loop. + Callback is appended to the list by default. Parameters ---------- callback : Callback The callback to be registered. - index : int - The index at which the callback should be registered. + index : int, optional + The index at which the callback should be registered. The default is None. + If it is None, append the callback to the list. """ + if index is None: + index = len(self._callbacks) self._callbacks.insert(index, callback) def _initialize_state(self) -> None: From 8b64d1e6b5198da6e487052d9d45d2ddfdab164e Mon Sep 17 00:00:00 2001 From: Eddie Bergman Date: Mon, 26 Jun 2023 09:34:03 +0200 Subject: [PATCH 06/57] fix: dtypes (#1044) --- smac/model/gaussian_process/kernels/base_kernels.py | 8 ++++---- smac/runhistory/runhistory.py | 6 +++--- smac/utils/multi_objective.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/smac/model/gaussian_process/kernels/base_kernels.py b/smac/model/gaussian_process/kernels/base_kernels.py index 044ad4fca..4dd6ff6bb 100644 --- a/smac/model/gaussian_process/kernels/base_kernels.py +++ b/smac/model/gaussian_process/kernels/base_kernels.py @@ -250,12 +250,12 @@ def _signature(self, func: Callable) -> Signature: def _set_active_dims(self, operate_on: np.ndarray | None = None) -> None: """Sets dimensions this kernel should work on.""" - if operate_on is not None and type(operate_on) in (list, np.ndarray): + if operate_on is not None and isinstance(operate_on, (list, np.ndarray)): if not isinstance(operate_on, np.ndarray): - raise TypeError("The argument `operate_on` needs to be of type np.ndarray but is %s" % type(operate_on)) + raise TypeError(f"The argument `operate_on` needs to be of type np.ndarray but is {type(operate_on)}") - if operate_on.dtype != int: - raise ValueError("The dtype of argument `operate_on` needs to be int, but is %s" % operate_on.dtype) + if not np.issubdtype(operate_on.dtype, np.integer): + raise ValueError(f"The dtype of `operate_on` needs to be np.integer, but is {operate_on.dtype}") self.operate_on = operate_on self._len_active = len(operate_on) diff --git a/smac/runhistory/runhistory.py b/smac/runhistory/runhistory.py index a30e1356e..aaf88889c 100644 --- a/smac/runhistory/runhistory.py +++ b/smac/runhistory/runhistory.py @@ -435,7 +435,7 @@ def get_min_cost(self, config: Configuration) -> float: cost = self._min_cost_per_config.get(config_id, np.nan) # type: ignore if self._n_objectives > 1: - assert type(cost) == list + assert isinstance(cost, list) assert self.multi_objective_algorithm is not None costs = normalize_costs(cost, self._objective_bounds) @@ -443,7 +443,7 @@ def get_min_cost(self, config: Configuration) -> float: # Note: We have to mean here because we already got the min cost return self.multi_objective_algorithm(costs) - assert type(cost) == float + assert isinstance(cost, float) return float(cost) def average_cost( @@ -847,7 +847,7 @@ def load(self, filename: str | Path, configspace: ConfigurationSpace) -> None: for entry in data["data"]: # Set n_objectives first if self._n_objectives == -1: - if isinstance(entry[4], float) or isinstance(entry[4], int): + if isinstance(entry[4], (float, int)): self._n_objectives = 1 else: self._n_objectives = len(entry[4]) diff --git a/smac/utils/multi_objective.py b/smac/utils/multi_objective.py index 2dec4da94..f959fe783 100644 --- a/smac/utils/multi_objective.py +++ b/smac/utils/multi_objective.py @@ -29,7 +29,7 @@ def normalize_costs( costs = [] for v, b in zip(values, bounds): - assert type(v) != list + assert not isinstance(v, list) p = v - b[0] q = b[1] - b[0] From 81d2f77d5462d68ce3860449d0bf855c26f24d77 Mon Sep 17 00:00:00 2001 From: Eddie Bergman Date: Mon, 26 Jun 2023 09:35:34 +0200 Subject: [PATCH 07/57] Fix(config_selector): Budget checking (#1039) --- smac/main/config_selector.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/smac/main/config_selector.py b/smac/main/config_selector.py index 76ace17f8..7b2053eee 100644 --- a/smac/main/config_selector.py +++ b/smac/main/config_selector.py @@ -266,14 +266,14 @@ def _collect_data(self) -> tuple[np.ndarray, np.ndarray, np.ndarray]: assert self._runhistory_encoder is not None # If we use a float value as a budget, we want to train the model only on the highest budget - available_budgets = [] - for run_key in self._runhistory: - budget = run_key.budget - if budget not in available_budgets: - available_budgets.append(run_key.budget) - - # Sort available budgets from highest to lowest budget - available_budgets = sorted(list(set(available_budgets)), reverse=True) # type: ignore + unique_budgets: set[float] = {run_key.budget for run_key in self._runhistory if run_key.budget is not None} + + available_budgets: list[float] | list[None] + if len(unique_budgets) > 0: + # Sort available budgets from highest to lowest budget + available_budgets = sorted(unique_budgets, reverse=True) + else: + available_budgets = [None] # Get #points per budget and if there are enough samples, then build a model for b in available_budgets: From 5aa921abf5ddfa968ff7480d0e1cb596efeea863 Mon Sep 17 00:00:00 2001 From: Eddie Bergman Date: Mon, 26 Jun 2023 09:36:23 +0200 Subject: [PATCH 08/57] Fix: target runner with partial func (#1045) --- smac/runner/target_function_runner.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/smac/runner/target_function_runner.py b/smac/runner/target_function_runner.py index 28837caa0..5431f5ed4 100644 --- a/smac/runner/target_function_runner.py +++ b/smac/runner/target_function_runner.py @@ -7,6 +7,7 @@ import math import time import traceback +from functools import partial import numpy as np from ConfigSpace import Configuration @@ -88,7 +89,17 @@ def __init__( @property def meta(self) -> dict[str, Any]: # noqa: D102 meta = super().meta - meta.update({"code": str(self._target_function.__code__.co_code)}) + + # Partial's don't have a __code__ attribute but are a convenient + # way a user might want to pass a function to SMAC, specifying + # keyword arguments. + f = self._target_function + if isinstance(f, partial): + f = f.func + meta.update({"code": str(f.__code__.co_code)}) + meta.update({"code-partial-args": repr(f)}) + else: + meta.update({"code": str(self._target_function.__code__.co_code)}) return meta From 83eaab5f9d2b3de17fa7f0a8d714b902246aa60b Mon Sep 17 00:00:00 2001 From: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> Date: Tue, 27 Jun 2023 17:41:51 +0200 Subject: [PATCH 09/57] Feature/improve documentation (#1028) * Experimental instruction for installing SMAC in windows via WSL * More detailed documentation regarding continuing runs * Fixes * Fixes * Fixes * Update docs/10_experimental.rst Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> * Fixes from PR * Fixes from PR * Fixes from PR * Fixes from PR * Fixes from PR --------- Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> --- CHANGELOG.md | 2 ++ docs/10_experimental.rst | 48 +++++++++++++++++++++++++++++ docs/1_installation.rst | 8 +++++ docs/advanced_usage/10_continue.rst | 19 +++++++++--- examples/1_basics/5_continue.py | 12 ++++++-- smac/facade/abstract_facade.py | 6 ++-- smac/main/smbo.py | 2 +- 7 files changed, 87 insertions(+), 10 deletions(-) create mode 100644 docs/10_experimental.rst diff --git a/CHANGELOG.md b/CHANGELOG.md index d7d565b22..a9493c38c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Improvements - Add an error when we get an empty dict data_to_scatter so that we can avoid an internal error caused in Dask precautiously +- Add experimental instruction for installing SMAC in Windows via a WSL. +- More detailed documentation regarding continuing runs. ## Bugfixes - Fix bug in the incumbent selection in the case that multi-fidelity is combined with multi-objective (#1019). diff --git a/docs/10_experimental.rst b/docs/10_experimental.rst new file mode 100644 index 000000000..a075498c4 --- /dev/null +++ b/docs/10_experimental.rst @@ -0,0 +1,48 @@ +Experimental +============ + +.. warning:: + This part is experimental and might not work in each case. If you would like to suggest any changes, please let us know. + + +Installation in Windows via WSL +------------------------------ + +SMAC can be installed in a WSL (Windows Subsystem for Linux) under Windows. + +**1) Install WSL under Windows** + +Install WSL under Windows. This SMAC installation workflow was tested with Ubuntu 18.04. For Ubuntu 20.04, +it has been observed that the SMAC installation results in a segmentation fault (core dumped). + +**2) Get Anaconda** + +Download an Anaconda Linux version to drive D under Windows, e.g. D:\\Anaconda3-2023.03-1-Linux-x86_64 + +In the WSL, Windows resources are mounted under /mnt: + +.. code:: bash + + cd /mnt/d + bash Anaconda3-2023.03-1-Linux-x86_64 + +Enter this command to create the environment variable: + +.. code:: bash + + export PATH="$PATH:/home/${USER}/anaconda3/bin + +Input 'python' to check if the installation was successful. + +**3) Install SMAC** + +Change to your home folder and install the general software there: + +.. code:: bash + + cd /home/${USER} + sudo apt-get install software-properties-common + sudo apt-get update + sudo apt-get install build-essential swig + conda install gxx_linux-64 gcc_linux-64 swig + curl https://raw.githubusercontent.com/automl/smac3/master/requirements.txt | xargs -n 1 -L 1 pip install diff --git a/docs/1_installation.rst b/docs/1_installation.rst index 54f910a44..7b8dd9c4b 100644 --- a/docs/1_installation.rst +++ b/docs/1_installation.rst @@ -68,3 +68,11 @@ You must have `conda >= 4.9` installed. To update conda or check your current co Read `SMAC feedstock `_ for more details. + +Windows via WSL (Experimental) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +SMAC can be installed under Windows in a WSL (Windows Subsystem for Linux). +You can find an instruction on how to do this here: :ref:`Experimental` +However, this is experimental and might not work in each case. +If you would like to suggest any changes, please let us know. diff --git a/docs/advanced_usage/10_continue.rst b/docs/advanced_usage/10_continue.rst index 8a254c4e5..e29c3f9c9 100644 --- a/docs/advanced_usage/10_continue.rst +++ b/docs/advanced_usage/10_continue.rst @@ -1,14 +1,23 @@ Continue ======== -SMAC automatically restores states where it left off if a run was interrupted or finished. To do so, it reads in old -files (derived from scenario's name, output_directory and seed) and sets the components. +SMAC can automatically restore states where it left off if a run was interrupted or prematurely finished. To do so, +it reads in old files (derived from scenario's name, output_directory and seed) and obtains the scenario information +of the previous run from those to continue the run. + +The behavior can be controlled by setting the parameter ``overwrite`` in the facade to True or False, respectively: + +* If set to True, SMAC overwrites the run results if a previous run is found that is consistent in the meta data with the current setup. +* If set to False and a previous run is found that + + * is consistent in the meta data, the run is continued. + * is not consistent in the meta data, the user is asked for the exact behaviour (overwrite completely or rename old run first). .. warning:: - If you changed any code and specified a name, SMAC will ask you whether you still want to resume or - delete the old run completely. If you did not specify a name, SMAC generates a new name and the old run is - not affected. + If you changed any code affecting the run's meta data and specified a name, SMAC will ask you whether you still + want to overwrite the old run or rename the old run first. If you did not specify a name, SMAC generates a new name + and the old run is not affected. Please have a look at our :ref:`continue example`. \ No newline at end of file diff --git a/examples/1_basics/5_continue.py b/examples/1_basics/5_continue.py index 04cca4aed..025856fee 100644 --- a/examples/1_basics/5_continue.py +++ b/examples/1_basics/5_continue.py @@ -2,8 +2,16 @@ Continue an Optimization ^^^^^^^^^^^^^^^^^^^^^^^^ -SMAC can also be continued. In this example, an optimization of a simple quadratic -function is continued. We use a custom callback, to artificially stop the first optimization. +SMAC can also be continued from a previous run. To do so, it reads in old files (derived from scenario's name, +output_directory and seed) and sets the corresponding components. In this example, an optimization of a simple quadratic +function is continued. + +First, after creating a scenario with 50 trials, we run SMAC with overwrite=True. This will +overwrite any previous runs (in case the example was called before). We use a custom callback to artificially stop +this first optimization after 10 trials. + +Second, we again run the SMAC optimization using the same scenario, but this time with overwrite=False. As +there already is a previous run with the same meta data, this run will be continued until the 50 trials are reached. """ from __future__ import annotations diff --git a/smac/facade/abstract_facade.py b/smac/facade/abstract_facade.py index 1022034c8..a5db5ab52 100644 --- a/smac/facade/abstract_facade.py +++ b/smac/facade/abstract_facade.py @@ -92,8 +92,10 @@ class AbstractFacade: Callbacks, which are incorporated into the optimization loop. overwrite: bool, defaults to False When True, overwrites the run results if a previous run is found that is - inconsistent in the meta data with the current setup. If ``overwrite`` is set to False, the user is asked - for the exact behaviour (overwrite completely, save old run, or use old results). + consistent in the meta data with the current setup. When False and a previous run is found that is + consistent in the meta data, the run is continued. When False and a previous run is found that is + not consistent in the meta data, the the user is asked for the exact behaviour (overwrite completely + or rename old run first). dask_client: Client | None, defaults to None User-created dask client, which can be used to start a dask cluster and then attach SMAC to it. This will not be closed automatically and will have to be closed manually if provided explicitly. If none is provided diff --git a/smac/main/smbo.py b/smac/main/smbo.py index a0ec60caa..6138e4d64 100644 --- a/smac/main/smbo.py +++ b/smac/main/smbo.py @@ -505,7 +505,7 @@ def _initialize_state(self) -> None: logger.info("Since the previous run was not successful, SMAC will start from scratch again.") self.reset() else: - # Here, we run into differen scenarios + # Here, we run into different scenarios diff = recursively_compare_dicts( Scenario.make_serializable(self._scenario), Scenario.make_serializable(old_scenario), From cbffa5f30d8f9bbe1916c7e3b5b0c71546f8134d Mon Sep 17 00:00:00 2001 From: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> Date: Thu, 20 Jul 2023 11:30:15 +0200 Subject: [PATCH 10/57] Handle configspace as dictionary in mlp example (#1057) * Handle configspace as dictionary in mlp example * Adapt sgd loss to newest scikit-learn version * Add change to changelog * Handle configspace as dictionary in parego example * Correct get usage --- CHANGELOG.md | 2 ++ examples/2_multi_fidelity/1_mlp_epochs.py | 6 +++--- examples/2_multi_fidelity/2_sgd_datasets.py | 2 +- examples/3_multi_objective/2_parego.py | 6 +++--- tests/fixtures/models.py | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9493c38c..dbcac80e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ## Bugfixes - Fix bug in the incumbent selection in the case that multi-fidelity is combined with multi-objective (#1019). - Fix callback order (#1040). +- Handle configspace as dictionary in mlp and parego example. +- Adapt sgd loss to newest scikit-learn version. # 2.0.1 diff --git a/examples/2_multi_fidelity/1_mlp_epochs.py b/examples/2_multi_fidelity/1_mlp_epochs.py index 9fd256c5d..48c027d3b 100644 --- a/examples/2_multi_fidelity/1_mlp_epochs.py +++ b/examples/2_multi_fidelity/1_mlp_epochs.py @@ -84,9 +84,9 @@ def train(self, config: Configuration, seed: int = 0, budget: int = 25) -> float # For deactivated parameters (by virtue of the conditions), # the configuration stores None-values. # This is not accepted by the MLP, so we replace them with placeholder values. - lr = config["learning_rate"] if config["learning_rate"] else "constant" - lr_init = config["learning_rate_init"] if config["learning_rate_init"] else 0.001 - batch_size = config["batch_size"] if config["batch_size"] else 200 + lr = config.get("learning_rate", "constant") + lr_init = config.get("learning_rate_init", 0.001) + batch_size = config.get("batch_size", 200) with warnings.catch_warnings(): warnings.filterwarnings("ignore") diff --git a/examples/2_multi_fidelity/2_sgd_datasets.py b/examples/2_multi_fidelity/2_sgd_datasets.py index 384d1c224..09864a963 100644 --- a/examples/2_multi_fidelity/2_sgd_datasets.py +++ b/examples/2_multi_fidelity/2_sgd_datasets.py @@ -89,7 +89,7 @@ def train(self, config: Configuration, instance: str, seed: int = 0) -> float: # SGD classifier using given configuration clf = SGDClassifier( - loss="log", + loss="log_loss", penalty="elasticnet", alpha=config["alpha"], l1_ratio=config["l1_ratio"], diff --git a/examples/3_multi_objective/2_parego.py b/examples/3_multi_objective/2_parego.py index d8fdc5ff8..856c2e857 100644 --- a/examples/3_multi_objective/2_parego.py +++ b/examples/3_multi_objective/2_parego.py @@ -66,9 +66,9 @@ def configspace(self) -> ConfigurationSpace: return cs def train(self, config: Configuration, seed: int = 0, budget: int = 10) -> dict[str, float]: - lr = config["learning_rate"] if config["learning_rate"] else "constant" - lr_init = config["learning_rate_init"] if config["learning_rate_init"] else 0.001 - batch_size = config["batch_size"] if config["batch_size"] else 200 + lr = config.get("learning_rate", "constant") + lr_init = config.get("learning_rate_init", 0.001) + batch_size = config.get("batch_size", 200) start_time = time.time() diff --git a/tests/fixtures/models.py b/tests/fixtures/models.py index 1321176f2..29f9bf152 100644 --- a/tests/fixtures/models.py +++ b/tests/fixtures/models.py @@ -71,7 +71,7 @@ def train(self, config: Configuration, instance: str = "0-1", budget: float = 1, # SGD classifier using given configuration clf = SGDClassifier( - loss="log", + loss="log_loss", penalty="elasticnet", alpha=config["alpha"], l1_ratio=config["l1_ratio"], From bf02a23fc97c563a1169ef867b292bd24e1b5b4d Mon Sep 17 00:00:00 2001 From: Sarah Krebs Date: Mon, 31 Jul 2023 17:53:07 +0200 Subject: [PATCH 11/57] Raise version number --- CITATION.cff | 2 +- Makefile | 2 +- benchmark/src/wrappers/v20.py | 2 +- docs/conf.py | 1 + smac/__init__.py | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 36435d6ec..a984b24c5 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,7 +9,7 @@ date-released: "2016-08-17" url: "https://automl.github.io/SMAC3/master/index.html" repository-code: "https://github.com/automl/SMAC3" -version: "2.0.1" +version: "2.0.2" type: "software" keywords: diff --git a/Makefile b/Makefile index f7727cf7f..ba7a4268b 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ SHELL := /bin/bash NAME := SMAC3 PACKAGE_NAME := smac -VERSION := 2.0.1 +VERSION := 2.0.2 DIR := "${CURDIR}" SOURCE_DIR := ${PACKAGE_NAME} diff --git a/benchmark/src/wrappers/v20.py b/benchmark/src/wrappers/v20.py index 8ed0864ec..9f2be56dc 100644 --- a/benchmark/src/wrappers/v20.py +++ b/benchmark/src/wrappers/v20.py @@ -6,7 +6,7 @@ class Version20(Wrapper): - supported_versions: list[str] = ["2.0.1"] + supported_versions: list[str] = ["2.0.2"] def __init__(self, task: Task, seed: int) -> None: super().__init__(task, seed) diff --git a/docs/conf.py b/docs/conf.py index d7b559888..198361fb9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ "version": version, "versions": { f"v{version}": "#", + "v2.0.1": "https://automl.github.io/SMAC3/v2.0.1/", "v2.0.0": "https://automl.github.io/SMAC3/v2.0.0/", "v2.0.0b1": "https://automl.github.io/SMAC3/v2.0.0b1/", "v2.0.0a2": "https://automl.github.io/SMAC3/v2.0.0a2/", diff --git a/smac/__init__.py b/smac/__init__.py index f3fe5805b..ab2781297 100644 --- a/smac/__init__.py +++ b/smac/__init__.py @@ -19,7 +19,7 @@ Copyright {datetime.date.today().strftime('%Y')}, Marius Lindauer, Katharina Eggensperger, Matthias Feurer, André Biedenkapp, Difan Deng, Carolin Benjamins, Tim Ruhkopf, René Sass and Frank Hutter""" -version = "2.0.1" +version = "2.0.2" try: From 0650ad52703afcee386d723dd21a7d634fc118af Mon Sep 17 00:00:00 2001 From: Sarah Krebs Date: Mon, 31 Jul 2023 18:03:35 +0200 Subject: [PATCH 12/57] Add missing period in docs. --- docs/1_installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/1_installation.rst b/docs/1_installation.rst index 7b8dd9c4b..a835d9cf1 100644 --- a/docs/1_installation.rst +++ b/docs/1_installation.rst @@ -73,6 +73,6 @@ Windows via WSL (Experimental) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SMAC can be installed under Windows in a WSL (Windows Subsystem for Linux). -You can find an instruction on how to do this here: :ref:`Experimental` +You can find an instruction on how to do this here: :ref:`Experimental`. However, this is experimental and might not work in each case. If you would like to suggest any changes, please let us know. From 541ee7e0383b491b86d1a23dcff669f2efad616d Mon Sep 17 00:00:00 2001 From: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> Date: Tue, 1 Aug 2023 15:02:51 +0200 Subject: [PATCH 13/57] Version 2.0.2 (#1060) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * maint * fix wrongly imported func name in tests * Add reference for pSMAC #609 * Rename example file * Added roadmap to readme * Prettification * Add result merger * Reformat with black * Delete unnecessary import * Fix typing * Inject first trajectory entry from file from first rundir * Add test for ResultMerger * Fix docstring * Get runhistory and trajectory from pSMAC run Via the result merger. Searchers rundirs and merges found runhistories and calculates a new trajectory. * Get trajectory * Add regex to requirements * Fixed MO cache problem * Fix MO tests * Fixing MO in AbstractRacer * Accepting ints * Hacking in the sum cost for MO * Fix mypi * Bugfix * Make mypi happy * Fix tests * Make format * Remove num_obj in MO optimizer * A lot of restructuring * Solve importings * Solve importings * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Updated changelog * Still fixing :) * fix docstrings * Change directory of tests * Fix ambiguous variable * Del newlines * fix mypy * mypy and doc fix * Removed int type checking * fix tests * Always return List of incs * fix path issue in example * maint boing facade * Better docstring * Return values if no bounds passed * Removed import * Return only one incumbent * Rename n_optimizers -> n_workers * Update example * Update docstring * Remove legacy function * Update to new signature * Update changelog.md * Format black * Update test * Please pre-commit * Merge development * Fix tests * Fix typos * Format black * Updated text * Increased version * Improved readme * SMAC logo 50% * SMAC logo 50% * Removed dev-2.0 * Added doc workflow for latest tag * Fixing docs latest tag * Fixing docs latest tag * First steps new intensifier * Make it work again * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * Test issues * FIX formatting * Link to old documentation * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * FIX formatting * Fix unit tests * Add annotation * corrected log prob for tophat prior * Removed configurations from callback * Fixed tests * Fix mypy * Save docs as latest version too * Enabled all tests again * Make version an env * Added last version to docs * Global variable fix * Added v before version number * New Intensifier Implementation (#888) - Completely reimplemented the intensifiers (including Successive Halving and Hyperband): All intensifiers support multi-fidelity, multi-objective and multi-threading by nature now. - Expected behaviour for ask-and-tell interface ensured (also for Successive Halving). - Continuing a run is now fully supported. - Added more examples. - Updated documentation based on new implementation. * First benchmark steps * Added a warning * Set xi to 0.01 * Incorporated walltime in trajectory * Working version of benchmark * Changed requirements * Changed xi back to 0.0 again * No duplicate budgets * Fixed bug which updated tracker wrongly * Bugfix: Delete correct bracket+stage * Fixed tests * Added costs to the trajectory * Changed initial design max ratio to 0.25 * Make format * Added more details to intensifier * Finished benchmark * Added benchmark source files * Fixed tests because of initial design changes * Improved runhistory API * Fixed typo (#894) * Merge doc fixes * Doc fixes (#893) Co-authored-by: timruhkopf Co-authored-by: Carolin Benjamins Co-authored-by: dengdifan Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs * Make format * Satisfied mypy * Enabled more tests * Increased version * Mypy * Mypy * Mypy * Fixed critical bug when using highest budget for inc selection * Added SH+HB comparison * Stuff * Updated report * Fixed docstring * Fixed image path * Removed vscode folder * Bugfixes * Doc fixes * Updated changelog * Make isort * Fix typos * Removed new line * Added alpha2 to docs selection * update description * Fix 531 correctly create integers in initial design This PR fixes a bug in which integer hyperparameters would not be created correctly in the initial design. This went unnoticed, except in the case where it was part of a condition and the function deactivate_inactive_hyperparameters was called. This PR fixes this bug by mapping the random float that was sampled by, for example and LHD or Sobol sequence, to the float representation of an integer by calling _transform and _inverse_transform of the integer hyperparameter. * Add contact paragraph to README. * Increase days until stale issues are closed by stalebot (#904) * Edit changelog * Add issue number * Closes #907 * Clarify origin of configurations (#908) * Random forest speed up (#903) Co-authored-by: René Sass * Add: workflow to update pre-commit versions (#874) Co-authored-by: René Sass * Add Git-Flow Workflow Specification * [DOC] acquisition function maximizers * Add stale reminder email action * Increase version number of ConfigSpace * Add guideline for updating copyright notice years to CONTRIBUTING.md * Update requirement for pyrfr (#920) * Update pre-commit: isort,flake8 version numbers * Format black * Fix mypy * Ac bench (#929) * Add Branin AC Task * Add ac optimization type * Nicer plotting * Add ac mode * Update ac task * Make new smac env name adjustable * Increase python version for smac 1.4 * Update benchmark * Update plotting * Goodby comments * Fix path * Fix 1.4 wrapper * Update benchmark results --------- Co-authored-by: Carolin Benjamins * [DOC] added documentation on the incumbent update * Feature/add version to make publish (#931) * Add version, correct package name and use bash in makefile * Increase version * Increase version in makefile, add places with versions to make publish * Fix typo * Transform integer feature names to strings (#930) * Transform int features to str * Fix pre-commit * Update version to 2.0.0 * Update version to 2.0.0 * Update CHANGELOG.md * Add information on deepcave to the readme * Correct doc version * fix(runhistory): set id of the sampled config (#951) * fix: Set config_id of config in runhistory * ci: Set dependabot to target development branch (#963) * feat: Option preventing SMAC setting up logging (#947) * Version 2.0.0 (#898) * Update runhistory2epm.py * pre_commit * Updated changelog * Fixed comment * Updated changelog * Some small improvements * Removed trailing space * Updated changelog * Restricted pynisher * Updated changelog * Link JMLR paper * Update bibtex entry Before: arxiv paper, now journal article. * Fixed citation in docs * Fixed callback typos * BOinG for mixed hyperparameter search space * fix format * maint * maint documentation * maint pre-commit * fix workflow * maint * Add dependabot for workflow files * Bump actions/setup-python from 2 to 4 (#849) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/checkout from 2 to 3 (#850) Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * maint * Updated stale bot * Updated stale options again * Fixed typing * Increased version * rename check_point_in_ss to check_subspace_point * Make format * Uncomment test procedure * docstrings for boing facade * avoid typing namespace everywhere in epm_configuration_chooser * fix docstring in rh2epm_boing * Limit on dask * Trying to fix tests * Replaced typing in all files * docstring for boing subspace * Changed feature requests to feature * allow acq_optimizer_local_kwargs to be passed to boing subspace without acq_optimizer_local * fix pre-commit * Added any * Update facade * Fix typing * Match arguments from pSMAC with SMAC4AC signature * Add description of pSMAC to docstring * Add option to pass facade class to pSMAC * Add example for pSMAC * Removed docstring in readme's example * Corrected text * move construct_gp_kernels to smac.epm_gpytorch.boing_kernels * maint * fix wrongly imported func name in tests * Add reference for pSMAC #609 * Rename example file * Added roadmap to readme * Prettification * Add result merger * Reformat with black * Delete unnecessary import * Fix typing * Inject first trajectory entry from file from first rundir * Add test for ResultMerger * Fix docstring * Get runhistory and trajectory from pSMAC run Via the result merger. Searchers rundirs and merges found runhistories and calculates a new trajectory. * Get trajectory * Add regex to requirements * Fixed MO cache problem * Fix MO tests * Fixing MO in AbstractRacer * Accepting ints * Hacking in the sum cost for MO * Fix mypi * Bugfix * Make mypi happy * Fix tests * Make format * Remove num_obj in MO optimizer * A lot of restructuring * Solve importings * Solve importings * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Updated changelog * Still fixing :) * fix docstrings * Change directory of tests * Fix ambiguous variable * Del newlines * fix mypy * mypy and doc fix * Removed int type checking * fix tests * Always return List of incs * fix path issue in example * maint boing facade * Better docstring * Return values if no bounds passed * Removed import * Return only one incumbent * Rename n_optimizers -> n_workers * Update example * Update docstring * Remove legacy function * Update to new signature * Update changelog.md * Format black * Update test * Please pre-commit * Merge development * Fix tests * Fix typos * Format black * Updated text * Increased version * Improved readme * SMAC logo 50% * SMAC logo 50% * Removed dev-2.0 * Added doc workflow for latest tag * Fixing docs latest tag * Fixing docs latest tag * First steps new intensifier * Make it work again * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * Test issues * FIX formatting * Link to old documentation * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * FIX formatting * Fix unit tests * Add annotation * corrected log prob for tophat prior * Removed configurations from callback * Fixed tests * Fix mypy * Save docs as latest version too * Enabled all tests again * Make version an env * Added last version to docs * Global variable fix * Added v before version number * New Intensifier Implementation (#888) - Completely reimplemented the intensifiers (including Successive Halving and Hyperband): All intensifiers support multi-fidelity, multi-objective and multi-threading by nature now. - Expected behaviour for ask-and-tell interface ensured (also for Successive Halving). - Continuing a run is now fully supported. - Added more examples. - Updated documentation based on new implementation. * First benchmark steps * Added a warning * Set xi to 0.01 * Incorporated walltime in trajectory * Working version of benchmark * Changed requirements * Changed xi back to 0.0 again * No duplicate budgets * Fixed bug which updated tracker wrongly * Bugfix: Delete correct bracket+stage * Fixed tests * Added costs to the trajectory * Changed initial design max ratio to 0.25 * Make format * Added more details to intensifier * Finished benchmark * Added benchmark source files * Fixed tests because of initial design changes * Improved runhistory API * Fixed typo (#894) * Merge doc fixes * Doc fixes (#893) Co-authored-by: timruhkopf Co-authored-by: Carolin Benjamins Co-authored-by: dengdifan Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs * Make format * Satisfied mypy * Enabled more tests * Increased version * Mypy * Mypy * Mypy * Fixed critical bug when using highest budget for inc selection * Added SH+HB comparison * Stuff * Updated report * Fixed docstring * Fixed image path * Removed vscode folder * Bugfixes * Doc fixes * Updated changelog * Make isort * Fix typos * Removed new line * Added alpha2 to docs selection * update description * Fix 531 correctly create integers in initial design This PR fixes a bug in which integer hyperparameters would not be created correctly in the initial design. This went unnoticed, except in the case where it was part of a condition and the function deactivate_inactive_hyperparameters was called. This PR fixes this bug by mapping the random float that was sampled by, for example and LHD or Sobol sequence, to the float representation of an integer by calling _transform and _inverse_transform of the integer hyperparameter. * Add contact paragraph to README. * Increase days until stale issues are closed by stalebot (#904) * Edit changelog * Add issue number * Closes #907 * Clarify origin of configurations (#908) * Random forest speed up (#903) Co-authored-by: René Sass * Add: workflow to update pre-commit versions (#874) Co-authored-by: René Sass * Add Git-Flow Workflow Specification * [DOC] acquisition function maximizers * Add stale reminder email action * Increase version number of ConfigSpace * Add guideline for updating copyright notice years to CONTRIBUTING.md * Update requirement for pyrfr (#920) * Update pre-commit: isort,flake8 version numbers * Format black * Fix mypy * Ac bench (#929) * Add Branin AC Task * Add ac optimization type * Nicer plotting * Add ac mode * Update ac task * Make new smac env name adjustable * Increase python version for smac 1.4 * Update benchmark * Update plotting * Goodby comments * Fix path * Fix 1.4 wrapper * Update benchmark results --------- Co-authored-by: Carolin Benjamins * [DOC] added documentation on the incumbent update * Feature/add version to make publish (#931) * Add version, correct package name and use bash in makefile * Increase version * Increase version in makefile, add places with versions to make publish * Fix typo * Transform integer feature names to strings (#930) * Transform int features to str * Fix pre-commit * Update version to 2.0.0 * Update version to 2.0.0 * Update CHANGELOG.md * Add information on deepcave to the readme * Correct doc version --------- Co-authored-by: dengdifan <33290713+dengdifan@users.noreply.github.com> Co-authored-by: dengdifan Co-authored-by: Matthias Feurer Co-authored-by: Carolin Benjamins Co-authored-by: eddiebergman Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Tim Ruhkopf * feat: Option preventing SMAC setting up logging * fix: Run isort * Bump peter-evans/create-pull-request from 3 to 5 (#958) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 3 to 5. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v3...v5) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/setup-python from 2 to 4 (#945) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat: Option preventing SMAC setting up logging * fix: Run isort * fix: Add to changelog * feat: Option preventing SMAC setting up logging * fix: Run isort --------- Signed-off-by: dependabot[bot] Co-authored-by: René Sass Co-authored-by: dengdifan <33290713+dengdifan@users.noreply.github.com> Co-authored-by: dengdifan Co-authored-by: Matthias Feurer Co-authored-by: Carolin Benjamins Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Tim Ruhkopf * Citation update (#961) * update CITATION.cff * update order in CITATION.cff * fix citation date * fix citation * maint * Adjust hyperband configuration distribution across brackets * Compute the actual differences between the isb keys. (#957) * Compute the actual differences between the isb keys. * Added change to CHANGELOG.md * Adjust schedule for stale reminder * fix(logging): Prevent automatic logging setup at init (#970) * fix(logging): Prevent automatic logging setup at init * fix(logging): Rely on `setup_logging` to handle arg * Fix validate method of smbo, update docstring. * Allow callbacks to be added to a specific index, make callback registering public * Update CHANGELOG.md * Fix broken references (#988) * Adapt developer install instructions to include pre-commit installation (#994) * [Feature] Initial Design now supports executing the default config provided by the user by means of the configspace he generated. (and is no longer ignored) * Pipe dask_client through facade for exec on hpc (#983) * Pipe dask_client through facade for exec on hpc * Handle pynisher's memory arg Before: Could only pass int via SMAC with the default unit B, but now tuple/list is possible * Update CHANGELOG.md * Make format * No printi --------- Co-authored-by: Carolin Benjamins * [Bug-fix] Pr was failing due to mutable additional configs default * [doc-fix] the count of initial design configs is computed slightly differently * Feature/metadata callback (#999) * Add example for using a callback to log run metadata to a file * Add issue number in changelog * Changelog formatting * Simplify metadata example * Simplify metadata example * Put callback in smac directory to allow importing it in custom repos * Move callback files to callback directory * Edit changelog * Correct metadata callback argument types * Additional metadata arguments in example * Set metadata default values in callback * Edit comment to fix PR * Documentation/dask client example (#1001) * Update requirements (dask_jobqueue) * Fix broken link * Ignore parallel example * Add parallelization example * Remove imports, add comment abt dask client * Add E's information * Update docs * Add comment about n_workers --------- Co-authored-by: Carolin Benjamins * Documentation/dask client example (#1001) * Update requirements (dask_jobqueue) * Fix broken link * Ignore parallel example * Add parallelization example * Remove imports, add comment abt dask client * Add E's information * Update docs * Add comment about n_workers --------- Co-authored-by: Carolin Benjamins * Feature/dask client (#1002) * Add info message that scenario.n_workers is ignored When passing a dask client, directly set n_workers in client. * Change info to warning --------- Co-authored-by: Carolin Benjamins * Update smac version (#1003) * Update smac version * Add hint on github release in make publish * Update copyright year * check if config in rh when storing state (#997) * check if config in rh when storing state * maint * remove unnecessary changes * Adapt developer install instructions to include pre-commit installation (#994) * Pipe dask_client through facade for exec on hpc (#983) * Pipe dask_client through facade for exec on hpc * Handle pynisher's memory arg Before: Could only pass int via SMAC with the default unit B, but now tuple/list is possible * Update CHANGELOG.md * Make format * No printi --------- Co-authored-by: Carolin Benjamins * update changelog * Increase tolerance for runhistory restoration test --------- Co-authored-by: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: Helena Graf * Add a workaround to be able to pass a dataset via dask.scatter (#993) * [feat] Support dask.scatter * [fix] Add shared_data arg in abstract_facade * Address the comments by Eddie * [fix] Fix errors raised by Eddie * [format] Apply black locally * [fix] Get retries back to 16 as it was a mistake * [doc] Update CHANGELOG --------- Co-authored-by: helegraf Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> * Adding Binder links for example (#976) * binder * maint * fix dependencies * maint docs * Revert "maint docs" This reverts commit 4fa1a0fae8dfa0b646e43273d50f72ddd76bf191. * add requirement --------- Co-authored-by: Difan Deng * Remove commit message from dependabot PRs (#1009) * Bugfix/fix broken link (#1012) * Remove commit message from dependabot PRs (#1010) * rename check_point_in_ss to check_subspace_point * Make format * Uncomment test procedure * docstrings for boing facade * avoid typing namespace everywhere in epm_configuration_chooser * fix docstring in rh2epm_boing * Limit on dask * Trying to fix tests * Replaced typing in all files * docstring for boing subspace * Changed feature requests to feature * allow acq_optimizer_local_kwargs to be passed to boing subspace without acq_optimizer_local * fix pre-commit * Added any * Update facade * Fix typing * Match arguments from pSMAC with SMAC4AC signature * Add description of pSMAC to docstring * Add option to pass facade class to pSMAC * Add example for pSMAC * Removed docstring in readme's example * Corrected text * move construct_gp_kernels to smac.epm_gpytorch.boing_kernels * maint * fix wrongly imported func name in tests * Add reference for pSMAC #609 * Rename example file * Added roadmap to readme * Prettification * Add result merger * Reformat with black * Delete unnecessary import * Fix typing * Inject first trajectory entry from file from first rundir * Add test for ResultMerger * Fix docstring * Get runhistory and trajectory from pSMAC run Via the result merger. Searchers rundirs and merges found runhistories and calculates a new trajectory. * Get trajectory * Add regex to requirements * Fixed MO cache problem * Fix MO tests * Fixing MO in AbstractRacer * Accepting ints * Hacking in the sum cost for MO * Fix mypi * Bugfix * Make mypi happy * Fix tests * Make format * Remove num_obj in MO optimizer * A lot of restructuring * Solve importings * Solve importings * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Updated changelog * Still fixing :) * fix docstrings * Change directory of tests * Fix ambiguous variable * Del newlines * fix mypy * mypy and doc fix * Removed int type checking * fix tests * Always return List of incs * fix path issue in example * maint boing facade * Better docstring * Return values if no bounds passed * Removed import * Return only one incumbent * Rename n_optimizers -> n_workers * Update example * Update docstring * Remove legacy function * Update to new signature * Update changelog.md * Format black * Update test * Please pre-commit * Merge development * Fix tests * Fix typos * Format black * Updated text * Increased version * Improved readme * SMAC logo 50% * SMAC logo 50% * Removed dev-2.0 * Added doc workflow for latest tag * Fixing docs latest tag * Fixing docs latest tag * First steps new intensifier * Make it work again * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * Test issues * FIX formatting * Link to old documentation * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * FIX formatting * Fix unit tests * Add annotation * corrected log prob for tophat prior * Removed configurations from callback * Fixed tests * Fix mypy * Save docs as latest version too * Enabled all tests again * Make version an env * Added last version to docs * Global variable fix * Added v before version number * New Intensifier Implementation (#888) - Completely reimplemented the intensifiers (including Successive Halving and Hyperband): All intensifiers support multi-fidelity, multi-objective and multi-threading by nature now. - Expected behaviour for ask-and-tell interface ensured (also for Successive Halving). - Continuing a run is now fully supported. - Added more examples. - Updated documentation based on new implementation. * First benchmark steps * Added a warning * Set xi to 0.01 * Incorporated walltime in trajectory * Working version of benchmark * Changed requirements * Changed xi back to 0.0 again * No duplicate budgets * Fixed bug which updated tracker wrongly * Bugfix: Delete correct bracket+stage * Fixed tests * Added costs to the trajectory * Changed initial design max ratio to 0.25 * Make format * Added more details to intensifier * Finished benchmark * Added benchmark source files * Fixed tests because of initial design changes * Improved runhistory API * Fixed typo (#894) * Merge doc fixes * Doc fixes (#893) Co-authored-by: timruhkopf Co-authored-by: Carolin Benjamins Co-authored-by: dengdifan Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs * Make format * Satisfied mypy * Enabled more tests * Increased version * Mypy * Mypy * Mypy * Fixed critical bug when using highest budget for inc selection * Added SH+HB comparison * Stuff * Updated report * Fixed docstring * Fixed image path * Removed vscode folder * Bugfixes * Doc fixes * Updated changelog * Make isort * Fix typos * Removed new line * Added alpha2 to docs selection * update description * Fix 531 correctly create integers in initial design This PR fixes a bug in which integer hyperparameters would not be created correctly in the initial design. This went unnoticed, except in the case where it was part of a condition and the function deactivate_inactive_hyperparameters was called. This PR fixes this bug by mapping the random float that was sampled by, for example and LHD or Sobol sequence, to the float representation of an integer by calling _transform and _inverse_transform of the integer hyperparameter. * Add contact paragraph to README. * Increase days until stale issues are closed by stalebot (#904) * Edit changelog * Add issue number * Closes #907 * Clarify origin of configurations (#908) * Random forest speed up (#903) Co-authored-by: René Sass * Add: workflow to update pre-commit versions (#874) Co-authored-by: René Sass * Add Git-Flow Workflow Specification * [DOC] acquisition function maximizers * Add stale reminder email action * Increase version number of ConfigSpace * Add guideline for updating copyright notice years to CONTRIBUTING.md * Update requirement for pyrfr (#920) * Update pre-commit: isort,flake8 version numbers * Format black * Fix mypy * Ac bench (#929) * Add Branin AC Task * Add ac optimization type * Nicer plotting * Add ac mode * Update ac task * Make new smac env name adjustable * Increase python version for smac 1.4 * Update benchmark * Update plotting * Goodby comments * Fix path * Fix 1.4 wrapper * Update benchmark results --------- Co-authored-by: Carolin Benjamins * [DOC] added documentation on the incumbent update * Feature/add version to make publish (#931) * Add version, correct package name and use bash in makefile * Increase version * Increase version in makefile, add places with versions to make publish * Fix typo * Transform integer feature names to strings (#930) * Transform int features to str * Fix pre-commit * Update version to 2.0.0 * Update version to 2.0.0 * Update CHANGELOG.md * Add information on deepcave to the readme * Correct doc version * fix(runhistory): set id of the sampled config (#951) * fix: Set config_id of config in runhistory * ci: Set dependabot to target development branch (#963) * feat: Option preventing SMAC setting up logging (#947) * Version 2.0.0 (#898) * Update runhistory2epm.py * pre_commit * Updated changelog * Fixed comment * Updated changelog * Some small improvements * Removed trailing space * Updated changelog * Restricted pynisher * Updated changelog * Link JMLR paper * Update bibtex entry Before: arxiv paper, now journal article. * Fixed citation in docs * Fixed callback typos * BOinG for mixed hyperparameter search space * fix format * maint * maint documentation * maint pre-commit * fix workflow * maint * Add dependabot for workflow files * Bump actions/setup-python from 2 to 4 (#849) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/checkout from 2 to 3 (#850) Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * maint * Updated stale bot * Updated stale options again * Fixed typing * Increased version * rename check_point_in_ss to check_subspace_point * Make format * Uncomment test procedure * docstrings for boing facade * avoid typing namespace everywhere in epm_configuration_chooser * fix docstring in rh2epm_boing * Limit on dask * Trying to fix tests * Replaced typing in all files * docstring for boing subspace * Changed feature requests to feature * allow acq_optimizer_local_kwargs to be passed to boing subspace without acq_optimizer_local * fix pre-commit * Added any * Update facade * Fix typing * Match arguments from pSMAC with SMAC4AC signature * Add description of pSMAC to docstring * Add option to pass facade class to pSMAC * Add example for pSMAC * Removed docstring in readme's example * Corrected text * move construct_gp_kernels to smac.epm_gpytorch.boing_kernels * maint * fix wrongly imported func name in tests * Add reference for pSMAC #609 * Rename example file * Added roadmap to readme * Prettification * Add result merger * Reformat with black * Delete unnecessary import * Fix typing * Inject first trajectory entry from file from first rundir * Add test for ResultMerger * Fix docstring * Get runhistory and trajectory from pSMAC run Via the result merger. Searchers rundirs and merges found runhistories and calculates a new trajectory. * Get trajectory * Add regex to requirements * Fixed MO cache problem * Fix MO tests * Fixing MO in AbstractRacer * Accepting ints * Hacking in the sum cost for MO * Fix mypi * Bugfix * Make mypi happy * Fix tests * Make format * Remove num_obj in MO optimizer * A lot of restructuring * Solve importings * Solve importings * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Updated changelog * Still fixing :) * fix docstrings * Change directory of tests * Fix ambiguous variable * Del newlines * fix mypy * mypy and doc fix * Removed int type checking * fix tests * Always return List of incs * fix path issue in example * maint boing facade * Better docstring * Return values if no bounds passed * Removed import * Return only one incumbent * Rename n_optimizers -> n_workers * Update example * Update docstring * Remove legacy function * Update to new signature * Update changelog.md * Format black * Update test * Please pre-commit * Merge development * Fix tests * Fix typos * Format black * Updated text * Increased version * Improved readme * SMAC logo 50% * SMAC logo 50% * Removed dev-2.0 * Added doc workflow for latest tag * Fixing docs latest tag * Fixing docs latest tag * First steps new intensifier * Make it work again * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * Test issues * FIX formatting * Link to old documentation * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * FIX formatting * Fix unit tests * Add annotation * corrected log prob for tophat prior * Removed configurations from callback * Fixed tests * Fix mypy * Save docs as latest version too * Enabled all tests again * Make version an env * Added last version to docs * Global variable fix * Added v before version number * New Intensifier Implementation (#888) - Completely reimplemented the intensifiers (including Successive Halving and Hyperband): All intensifiers support multi-fidelity, multi-objective and multi-threading by nature now. - Expected behaviour for ask-and-tell interface ensured (also for Successive Halving). - Continuing a run is now fully supported. - Added more examples. - Updated documentation based on new implementation. * First benchmark steps * Added a warning * Set xi to 0.01 * Incorporated walltime in trajectory * Working version of benchmark * Changed requirements * Changed xi back to 0.0 again * No duplicate budgets * Fixed bug which updated tracker wrongly * Bugfix: Delete correct bracket+stage * Fixed tests * Added costs to the trajectory * Changed initial design max ratio to 0.25 * Make format * Added more details to intensifier * Finished benchmark * Added benchmark source files * Fixed tests because of initial design changes * Improved runhistory API * Fixed typo (#894) * Merge doc fixes * Doc fixes (#893) Co-authored-by: timruhkopf Co-authored-by: Carolin Benjamins Co-authored-by: dengdifan Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs * Make format * Satisfied mypy * Enabled more tests * Increased version * Mypy * Mypy * Mypy * Fixed critical bug when using highest budget for inc selection * Added SH+HB comparison * Stuff * Updated report * Fixed docstring * Fixed image path * Removed vscode folder * Bugfixes * Doc fixes * Updated changelog * Make isort * Fix typos * Removed new line * Added alpha2 to docs selection * update description * Fix 531 correctly create integers in initial design This PR fixes a bug in which integer hyperparameters would not be created correctly in the initial design. This went unnoticed, except in the case where it was part of a condition and the function deactivate_inactive_hyperparameters was called. This PR fixes this bug by mapping the random float that was sampled by, for example and LHD or Sobol sequence, to the float representation of an integer by calling _transform and _inverse_transform of the integer hyperparameter. * Add contact paragraph to README. * Increase days until stale issues are closed by stalebot (#904) * Edit changelog * Add issue number * Closes #907 * Clarify origin of configurations (#908) * Random forest speed up (#903) Co-authored-by: René Sass * Add: workflow to update pre-commit versions (#874) Co-authored-by: René Sass * Add Git-Flow Workflow Specification * [DOC] acquisition function maximizers * Add stale reminder email action * Increase version number of ConfigSpace * Add guideline for updating copyright notice years to CONTRIBUTING.md * Update requirement for pyrfr (#920) * Update pre-commit: isort,flake8 version numbers * Format black * Fix mypy * Ac bench (#929) * Add Branin AC Task * Add ac optimization type * Nicer plotting * Add ac mode * Update ac task * Make new smac env name adjustable * Increase python version for smac 1.4 * Update benchmark * Update plotting * Goodby comments * Fix path * Fix 1.4 wrapper * Update benchmark results --------- Co-authored-by: Carolin Benjamins * [DOC] added documentation on the incumbent update * Feature/add version to make publish (#931) * Add version, correct package name and use bash in makefile * Increase version * Increase version in makefile, add places with versions to make publish * Fix typo * Transform integer feature names to strings (#930) * Transform int features to str * Fix pre-commit * Update version to 2.0.0 * Update version to 2.0.0 * Update CHANGELOG.md * Add information on deepcave to the readme * Correct doc version --------- Co-authored-by: dengdifan <33290713+dengdifan@users.noreply.github.com> Co-authored-by: dengdifan Co-authored-by: Matthias Feurer Co-authored-by: Carolin Benjamins Co-authored-by: eddiebergman Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Tim Ruhkopf * feat: Option preventing SMAC setting up logging * fix: Run isort * Bump peter-evans/create-pull-request from 3 to 5 (#958) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 3 to 5. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v3...v5) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/setup-python from 2 to 4 (#945) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat: Option preventing SMAC setting up logging * fix: Run isort * fix: Add to changelog * feat: Option preventing SMAC setting up logging * fix: Run isort --------- Signed-off-by: dependabot[bot] Co-authored-by: René Sass Co-authored-by: dengdifan <33290713+dengdifan@users.noreply.github.com> Co-authored-by: dengdifan Co-authored-by: Matthias Feurer Co-authored-by: Carolin Benjamins Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Tim Ruhkopf * Citation update (#961) * update CITATION.cff * update order in CITATION.cff * fix citation date * fix citation * maint * Adjust hyperband configuration distribution across brackets * Compute the actual differences between the isb keys. (#957) * Compute the actual differences between the isb keys. * Added change to CHANGELOG.md * Adjust schedule for stale reminder * fix(logging): Prevent automatic logging setup at init (#970) * fix(logging): Prevent automatic logging setup at init * fix(logging): Rely on `setup_logging` to handle arg * Fix validate method of smbo, update docstring. * Allow callbacks to be added to a specific index, make callback registering public * Update CHANGELOG.md * Fix broken references (#988) * Adapt developer install instructions to include pre-commit installation (#994) * [Feature] Initial Design now supports executing the default config provided by the user by means of the configspace he generated. (and is no longer ignored) * Pipe dask_client through facade for exec on hpc (#983) * Pipe dask_client through facade for exec on hpc * Handle pynisher's memory arg Before: Could only pass int via SMAC with the default unit B, but now tuple/list is possible * Update CHANGELOG.md * Make format * No printi --------- Co-authored-by: Carolin Benjamins * [Bug-fix] Pr was failing due to mutable additional configs default * [doc-fix] the count of initial design configs is computed slightly differently * Feature/metadata callback (#999) * Add example for using a callback to log run metadata to a file * Add issue number in changelog * Changelog formatting * Simplify metadata example * Simplify metadata example * Put callback in smac directory to allow importing it in custom repos * Move callback files to callback directory * Edit changelog * Correct metadata callback argument types * Additional metadata arguments in example * Set metadata default values in callback * Edit comment to fix PR * Documentation/dask client example (#1001) * Update requirements (dask_jobqueue) * Fix broken link * Ignore parallel example * Add parallelization example * Remove imports, add comment abt dask client * Add E's information * Update docs * Add comment about n_workers --------- Co-authored-by: Carolin Benjamins * Documentation/dask client example (#1001) * Update requirements (dask_jobqueue) * Fix broken link * Ignore parallel example * Add parallelization example * Remove imports, add comment abt dask client * Add E's information * Update docs * Add comment about n_workers --------- Co-authored-by: Carolin Benjamins * Feature/dask client (#1002) * Add info message that scenario.n_workers is ignored When passing a dask client, directly set n_workers in client. * Change info to warning --------- Co-authored-by: Carolin Benjamins * Update smac version (#1003) * Update smac version * Add hint on github release in make publish * Update copyright year * check if config in rh when storing state (#997) * check if config in rh when storing state * maint * remove unnecessary changes * Adapt developer install instructions to include pre-commit installation (#994) * Pipe dask_client through facade for exec on hpc (#983) * Pipe dask_client through facade for exec on hpc * Handle pynisher's memory arg Before: Could only pass int via SMAC with the default unit B, but now tuple/list is possible * Update CHANGELOG.md * Make format * No printi --------- Co-authored-by: Carolin Benjamins * update changelog * Increase tolerance for runhistory restoration test --------- Co-authored-by: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: Helena Graf * Add a workaround to be able to pass a dataset via dask.scatter (#993) * [feat] Support dask.scatter * [fix] Add shared_data arg in abstract_facade * Address the comments by Eddie * [fix] Fix errors raised by Eddie * [format] Apply black locally * [fix] Get retries back to 16 as it was a mistake * [doc] Update CHANGELOG --------- Co-authored-by: helegraf Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> * Adding Binder links for example (#976) * binder * maint * fix dependencies * maint docs * Revert "maint docs" This reverts commit 4fa1a0fae8dfa0b646e43273d50f72ddd76bf191. * add requirement --------- Co-authored-by: Difan Deng * Remove commit message from dependabot PRs (#1009) --------- Signed-off-by: dependabot[bot] Co-authored-by: dengdifan Co-authored-by: René Sass Co-authored-by: Carolin Benjamins Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Matthias Feurer Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Eddie Bergman Co-authored-by: Tim Ruhkopf Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> Co-authored-by: Aron Bahram Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Co-authored-by: Difan Deng * Fix broken link --------- Signed-off-by: dependabot[bot] Co-authored-by: dengdifan Co-authored-by: René Sass Co-authored-by: Carolin Benjamins Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Matthias Feurer Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Eddie Bergman Co-authored-by: Tim Ruhkopf Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> Co-authored-by: Aron Bahram Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Co-authored-by: Difan Deng * Drop torch requirements (#1015) * Remove torch requirements * Fix format --------- Co-authored-by: Carolin Benjamins * Remove leftover CLI entry points (#1014) * Fix incumbent selection in case of SH+MO * Update CHANGELOG.md * Feature/GitHub actions pr draft (#1008) Update the workflows only to trigger extensive tests for PRs that are not in the draft stage --------- Co-authored-by: timruhkopf Co-authored-by: Helena Graf * Update stale report to include issues with recent interactions (#1025) * Add recently updated issues to list of reminders. * Add recently updated issues to list of reminders. * Add recently updated issues to list of reminders. * Raise error if resource limitation and parallelization is requested (#1023) * Add warnings for parallelism doc * Check for parallelization + pynisher Does not work together, throw error. Does not work bc Dask pickles jobs and pynisher runs the target function in a process. * Fix whitespaces --------- Co-authored-by: Carolin Benjamins * Raise an error for invalid scatter data (#1018) * [enhance] Add an error for scatter data * Update ChangeLog --------- Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> * fix(configspace): shows all keys in print (#1043) * Fix callback order (#1040) Co-authored-by: Carolin Benjamins * fix: dtypes (#1044) * Fix(config_selector): Budget checking (#1039) * Fix: target runner with partial func (#1045) * Feature/improve documentation (#1028) * Experimental instruction for installing SMAC in windows via WSL * More detailed documentation regarding continuing runs * Fixes * Fixes * Fixes * Update docs/10_experimental.rst Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> * Fixes from PR * Fixes from PR * Fixes from PR * Fixes from PR * Fixes from PR --------- Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> * Handle configspace as dictionary in mlp example (#1057) * Handle configspace as dictionary in mlp example * Adapt sgd loss to newest scikit-learn version * Add change to changelog * Handle configspace as dictionary in parego example * Correct get usage * Raise version number * Add missing period in docs. --------- Signed-off-by: dependabot[bot] Co-authored-by: dengdifan Co-authored-by: Carolin Benjamins Co-authored-by: René Sass Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Matthias Feurer Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Eddie Bergman Co-authored-by: Tim Ruhkopf Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> Co-authored-by: Aron Bahram Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Co-authored-by: Difan Deng Co-authored-by: Bastian Zimmermann <10774221+BastianZim@users.noreply.github.com> --- .github/workflows/dist.yml | 10 +++- .github/workflows/docs.yml | 14 ++++-- .github/workflows/pre-commit.yml | 8 +++- .github/workflows/pytest.yml | 27 +++++++---- CHANGELOG.md | 13 +++++ CITATION.cff | 2 +- Makefile | 2 +- benchmark/src/wrappers/v20.py | 2 +- docs/10_experimental.rst | 48 +++++++++++++++++++ docs/1_installation.rst | 8 ++++ docs/advanced_usage/10_continue.rst | 19 ++++++-- docs/advanced_usage/9_parallelism.rst | 33 +++++++++++++ docs/conf.py | 1 + examples/1_basics/5_continue.py | 12 ++++- examples/2_multi_fidelity/1_mlp_epochs.py | 6 +-- examples/2_multi_fidelity/2_sgd_datasets.py | 2 +- examples/3_multi_objective/2_parego.py | 6 +-- .../3_metadata_callback.py | 3 +- setup.py | 6 --- smac/__init__.py | 2 +- smac/facade/abstract_facade.py | 21 +++++++- smac/intensifier/successive_halving.py | 3 +- smac/main/config_selector.py | 16 +++---- smac/main/smbo.py | 12 +++-- .../gaussian_process/kernels/base_kernels.py | 8 ++-- smac/runhistory/runhistory.py | 6 +-- smac/runner/target_function_runner.py | 13 ++++- smac/utils/configspace.py | 17 ++++--- smac/utils/multi_objective.py | 2 +- tests/fixtures/models.py | 2 +- tests/fixtures/scenario.py | 2 +- .../test_initial_design.py | 2 +- 32 files changed, 254 insertions(+), 74 deletions(-) create mode 100644 docs/10_experimental.rst diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index ffac9f005..0965ea2f1 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -1,7 +1,7 @@ name: dist on: - # Manual trigger option in github + # Manual trigger option in GitHub workflow_dispatch: # Trigger on push to these branches @@ -10,14 +10,20 @@ on: - main - development - # Trigger on a open/push to a PR targeting one of these branches + # Trigger on open/push to a PR targeting one of these branches pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review branches: - main - development jobs: dist: + if: ${{ !github.event.pull_request.draft }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e1b16334d..d89791712 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,9 +1,9 @@ name: docs on: - # Manual trigger option in github - # This won't push to github pages where docs are hosted due - # to the gaurded if statement in those steps + # Manual trigger option in GitHub + # This won't push to GitHub pages where docs are hosted due + # to the guarded if statement in those steps workflow_dispatch: # Trigger on push to these branches @@ -12,8 +12,13 @@ on: - main - development - # Trigger on a open/push to a PR targeting one of these branches + # Trigger on open/push to a PR targeting one of these branches pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review branches: - main - development @@ -23,6 +28,7 @@ env: jobs: build-and-deploy: + if: ${{ !github.event.pull_request.draft }} runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 5f5c400fc..e9ba1a01b 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -1,7 +1,7 @@ name: pre-commit on: - # Manually triggerable in github + # Manually triggerable in GitHub workflow_dispatch: # When a push occurs on either of these branches @@ -12,12 +12,18 @@ on: # When a push occurs on a PR that targets these branches pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review branches: - main - development jobs: run-all-files: + if : ${{ !github.event.pull_request.draft }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index f82e49c1d..37e15a472 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,7 +1,7 @@ name: tests on: - # Allow to manually trigger through github API + # Allow to manually trigger through GitHub API workflow_dispatch: # Triggers with push to main @@ -12,6 +12,11 @@ on: # Triggers with push to a pr aimed at main pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review branches: - main - development @@ -34,6 +39,7 @@ jobs: # General unit tests source-test: name: ${{ matrix.python-version }}-${{ matrix.os }} + if: ${{ !github.event.pull_request.draft }} runs-on: ${{ matrix.os }} defaults: @@ -44,7 +50,6 @@ jobs: fail-fast: false matrix: python-version: ["3.8", "3.9", "3.10"] - # python-version: ["3.8"] os: ["ubuntu-latest"] steps: @@ -70,10 +75,10 @@ jobs: - name: Tests run: | - pytest ${{ env.pytest-args }} ${{ env.test-dir }} - - # Post-hoc clean-up - make clean + echo "Running all tests..." + pytest ${{ env.pytest-args }} ${{ env.test-dir }} + # Post-hoc clean-up + make clean - name: Check for files left behind by test run: | @@ -89,6 +94,7 @@ jobs: # Testing with conda conda-tests: name: conda-${{ matrix.python-version }}-${{ matrix.os }} + if: ${{ !github.event.pull_request.draft }} runs-on: ${{ matrix.os }} defaults: run: @@ -98,7 +104,6 @@ jobs: fail-fast: false matrix: python-version: ["3.8", "3.9", "3.10"] - # python-version: ["3.8"] os: ["ubuntu-latest"] steps: @@ -120,11 +125,15 @@ jobs: - name: Tests run: | - pytest ${{ env.pytest-args }} ${{ env.test-dir }} + echo "Running all tests..." + pytest ${{ env.pytest-args }} ${{ env.test-dir }} + + # Testing a dist install dist-test: name: dist-${{ matrix.python-version }}-${{ matrix.os }} + if: ${{ !github.event.pull_request.draft }} runs-on: ${{ matrix.os }} defaults: @@ -135,7 +144,6 @@ jobs: fail-fast: false matrix: python-version: ["3.8", "3.9", "3.10"] - # python-version: ["3.8"] os: ["ubuntu-latest"] steps: @@ -161,4 +169,5 @@ jobs: - name: Tests run: | + echo "Running all tests..." pytest ${{ env.pytest-args }} ${{ env.test-dir }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 807cbaf7e..dbcac80e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +# 2.0.2 + +## Improvements +- Add an error when we get an empty dict data_to_scatter so that we can avoid an internal error caused in Dask precautiously +- Add experimental instruction for installing SMAC in Windows via a WSL. +- More detailed documentation regarding continuing runs. + +## Bugfixes +- Fix bug in the incumbent selection in the case that multi-fidelity is combined with multi-objective (#1019). +- Fix callback order (#1040). +- Handle configspace as dictionary in mlp and parego example. +- Adapt sgd loss to newest scikit-learn version. + # 2.0.1 ## Improvements diff --git a/CITATION.cff b/CITATION.cff index 36435d6ec..a984b24c5 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,7 +9,7 @@ date-released: "2016-08-17" url: "https://automl.github.io/SMAC3/master/index.html" repository-code: "https://github.com/automl/SMAC3" -version: "2.0.1" +version: "2.0.2" type: "software" keywords: diff --git a/Makefile b/Makefile index f7727cf7f..ba7a4268b 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ SHELL := /bin/bash NAME := SMAC3 PACKAGE_NAME := smac -VERSION := 2.0.1 +VERSION := 2.0.2 DIR := "${CURDIR}" SOURCE_DIR := ${PACKAGE_NAME} diff --git a/benchmark/src/wrappers/v20.py b/benchmark/src/wrappers/v20.py index 8ed0864ec..9f2be56dc 100644 --- a/benchmark/src/wrappers/v20.py +++ b/benchmark/src/wrappers/v20.py @@ -6,7 +6,7 @@ class Version20(Wrapper): - supported_versions: list[str] = ["2.0.1"] + supported_versions: list[str] = ["2.0.2"] def __init__(self, task: Task, seed: int) -> None: super().__init__(task, seed) diff --git a/docs/10_experimental.rst b/docs/10_experimental.rst new file mode 100644 index 000000000..a075498c4 --- /dev/null +++ b/docs/10_experimental.rst @@ -0,0 +1,48 @@ +Experimental +============ + +.. warning:: + This part is experimental and might not work in each case. If you would like to suggest any changes, please let us know. + + +Installation in Windows via WSL +------------------------------ + +SMAC can be installed in a WSL (Windows Subsystem for Linux) under Windows. + +**1) Install WSL under Windows** + +Install WSL under Windows. This SMAC installation workflow was tested with Ubuntu 18.04. For Ubuntu 20.04, +it has been observed that the SMAC installation results in a segmentation fault (core dumped). + +**2) Get Anaconda** + +Download an Anaconda Linux version to drive D under Windows, e.g. D:\\Anaconda3-2023.03-1-Linux-x86_64 + +In the WSL, Windows resources are mounted under /mnt: + +.. code:: bash + + cd /mnt/d + bash Anaconda3-2023.03-1-Linux-x86_64 + +Enter this command to create the environment variable: + +.. code:: bash + + export PATH="$PATH:/home/${USER}/anaconda3/bin + +Input 'python' to check if the installation was successful. + +**3) Install SMAC** + +Change to your home folder and install the general software there: + +.. code:: bash + + cd /home/${USER} + sudo apt-get install software-properties-common + sudo apt-get update + sudo apt-get install build-essential swig + conda install gxx_linux-64 gcc_linux-64 swig + curl https://raw.githubusercontent.com/automl/smac3/master/requirements.txt | xargs -n 1 -L 1 pip install diff --git a/docs/1_installation.rst b/docs/1_installation.rst index 54f910a44..a835d9cf1 100644 --- a/docs/1_installation.rst +++ b/docs/1_installation.rst @@ -68,3 +68,11 @@ You must have `conda >= 4.9` installed. To update conda or check your current co Read `SMAC feedstock `_ for more details. + +Windows via WSL (Experimental) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +SMAC can be installed under Windows in a WSL (Windows Subsystem for Linux). +You can find an instruction on how to do this here: :ref:`Experimental`. +However, this is experimental and might not work in each case. +If you would like to suggest any changes, please let us know. diff --git a/docs/advanced_usage/10_continue.rst b/docs/advanced_usage/10_continue.rst index 8a254c4e5..e29c3f9c9 100644 --- a/docs/advanced_usage/10_continue.rst +++ b/docs/advanced_usage/10_continue.rst @@ -1,14 +1,23 @@ Continue ======== -SMAC automatically restores states where it left off if a run was interrupted or finished. To do so, it reads in old -files (derived from scenario's name, output_directory and seed) and sets the components. +SMAC can automatically restore states where it left off if a run was interrupted or prematurely finished. To do so, +it reads in old files (derived from scenario's name, output_directory and seed) and obtains the scenario information +of the previous run from those to continue the run. + +The behavior can be controlled by setting the parameter ``overwrite`` in the facade to True or False, respectively: + +* If set to True, SMAC overwrites the run results if a previous run is found that is consistent in the meta data with the current setup. +* If set to False and a previous run is found that + + * is consistent in the meta data, the run is continued. + * is not consistent in the meta data, the user is asked for the exact behaviour (overwrite completely or rename old run first). .. warning:: - If you changed any code and specified a name, SMAC will ask you whether you still want to resume or - delete the old run completely. If you did not specify a name, SMAC generates a new name and the old run is - not affected. + If you changed any code affecting the run's meta data and specified a name, SMAC will ask you whether you still + want to overwrite the old run or rename the old run first. If you did not specify a name, SMAC generates a new name + and the old run is not affected. Please have a look at our :ref:`continue example`. \ No newline at end of file diff --git a/docs/advanced_usage/9_parallelism.rst b/docs/advanced_usage/9_parallelism.rst index 9912c782f..67e555215 100644 --- a/docs/advanced_usage/9_parallelism.rst +++ b/docs/advanced_usage/9_parallelism.rst @@ -21,6 +21,39 @@ SMAC supports multiple workers natively via Dask. Just specify ``n_workers`` in When using multiple workers, SMAC is not reproducible anymore. +.. warning :: + + You cannot use resource limitation (pynisher, via the `scenario` arguments `trail_walltime_limit` and `trial_memory_limit`). + This is because pynisher works by running your function inside of a subprocess. + Once in the subprocess, the resources will be limited for that process before running your function. + This does not work together with pickling - which is required by dask to schedule jobs on the cluster, even on a local one. + + +.. warning :: + + Start/run SMAC inside ``if __name__ == "__main__"`` in your script otherwise Dask is not able to correctly + spawn jobs and probably this runtime error will be raised: + + .. code-block :: + + RuntimeError: + An attempt has been made to start a new process before the + current process has finished its bootstrapping phase. + + This probably means that you are not using fork to start your + child processes and you have forgotten to use the proper idiom + in the main module: + + if __name__ == '__main__': + freeze_support() + ... + + The "freeze_support()" line can be omitted if the program + is not going to be frozen to produce an executable. + + + + Running on a Cluster -------------------- You can also pass a custom dask client, e.g. to run on a slurm cluster. diff --git a/docs/conf.py b/docs/conf.py index d7b559888..198361fb9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ "version": version, "versions": { f"v{version}": "#", + "v2.0.1": "https://automl.github.io/SMAC3/v2.0.1/", "v2.0.0": "https://automl.github.io/SMAC3/v2.0.0/", "v2.0.0b1": "https://automl.github.io/SMAC3/v2.0.0b1/", "v2.0.0a2": "https://automl.github.io/SMAC3/v2.0.0a2/", diff --git a/examples/1_basics/5_continue.py b/examples/1_basics/5_continue.py index 04cca4aed..025856fee 100644 --- a/examples/1_basics/5_continue.py +++ b/examples/1_basics/5_continue.py @@ -2,8 +2,16 @@ Continue an Optimization ^^^^^^^^^^^^^^^^^^^^^^^^ -SMAC can also be continued. In this example, an optimization of a simple quadratic -function is continued. We use a custom callback, to artificially stop the first optimization. +SMAC can also be continued from a previous run. To do so, it reads in old files (derived from scenario's name, +output_directory and seed) and sets the corresponding components. In this example, an optimization of a simple quadratic +function is continued. + +First, after creating a scenario with 50 trials, we run SMAC with overwrite=True. This will +overwrite any previous runs (in case the example was called before). We use a custom callback to artificially stop +this first optimization after 10 trials. + +Second, we again run the SMAC optimization using the same scenario, but this time with overwrite=False. As +there already is a previous run with the same meta data, this run will be continued until the 50 trials are reached. """ from __future__ import annotations diff --git a/examples/2_multi_fidelity/1_mlp_epochs.py b/examples/2_multi_fidelity/1_mlp_epochs.py index 9fd256c5d..48c027d3b 100644 --- a/examples/2_multi_fidelity/1_mlp_epochs.py +++ b/examples/2_multi_fidelity/1_mlp_epochs.py @@ -84,9 +84,9 @@ def train(self, config: Configuration, seed: int = 0, budget: int = 25) -> float # For deactivated parameters (by virtue of the conditions), # the configuration stores None-values. # This is not accepted by the MLP, so we replace them with placeholder values. - lr = config["learning_rate"] if config["learning_rate"] else "constant" - lr_init = config["learning_rate_init"] if config["learning_rate_init"] else 0.001 - batch_size = config["batch_size"] if config["batch_size"] else 200 + lr = config.get("learning_rate", "constant") + lr_init = config.get("learning_rate_init", 0.001) + batch_size = config.get("batch_size", 200) with warnings.catch_warnings(): warnings.filterwarnings("ignore") diff --git a/examples/2_multi_fidelity/2_sgd_datasets.py b/examples/2_multi_fidelity/2_sgd_datasets.py index 384d1c224..09864a963 100644 --- a/examples/2_multi_fidelity/2_sgd_datasets.py +++ b/examples/2_multi_fidelity/2_sgd_datasets.py @@ -89,7 +89,7 @@ def train(self, config: Configuration, instance: str, seed: int = 0) -> float: # SGD classifier using given configuration clf = SGDClassifier( - loss="log", + loss="log_loss", penalty="elasticnet", alpha=config["alpha"], l1_ratio=config["l1_ratio"], diff --git a/examples/3_multi_objective/2_parego.py b/examples/3_multi_objective/2_parego.py index d8fdc5ff8..856c2e857 100644 --- a/examples/3_multi_objective/2_parego.py +++ b/examples/3_multi_objective/2_parego.py @@ -66,9 +66,9 @@ def configspace(self) -> ConfigurationSpace: return cs def train(self, config: Configuration, seed: int = 0, budget: int = 10) -> dict[str, float]: - lr = config["learning_rate"] if config["learning_rate"] else "constant" - lr_init = config["learning_rate_init"] if config["learning_rate_init"] else 0.001 - batch_size = config["batch_size"] if config["batch_size"] else 200 + lr = config.get("learning_rate", "constant") + lr_init = config.get("learning_rate_init", 0.001) + batch_size = config.get("batch_size", 200) start_time = time.time() diff --git a/examples/4_advanced_optimizer/3_metadata_callback.py b/examples/4_advanced_optimizer/3_metadata_callback.py index 1f7078f94..ab35f2862 100644 --- a/examples/4_advanced_optimizer/3_metadata_callback.py +++ b/examples/4_advanced_optimizer/3_metadata_callback.py @@ -19,6 +19,7 @@ """ import sys + from ConfigSpace import Configuration, ConfigurationSpace, Float from smac import HyperparameterOptimizationFacade as HPOFacade @@ -65,7 +66,7 @@ def train(self, config: Configuration, seed: int = 0) -> float: branch="Name of Active Branch", commit="Commit Hash", command=" ".join(sys.argv), - additional_information="Some Additional Information" + additional_information="Some Additional Information", ) ], logging_level=999999, diff --git a/setup.py b/setup.py index 79249cd5f..f194d3677 100644 --- a/setup.py +++ b/setup.py @@ -20,9 +20,7 @@ def read_file(filepath: str) -> str: return fh.read() -torch_requirements = ["torch>=1.9.0", "gpytorch>=1.5.0", "pyro-ppl>=1.7.0", "botorch>=0.5.0"] extras_require = { - "gpytorch": torch_requirements, "dev": [ "setuptools", "types-setuptools", @@ -40,7 +38,6 @@ def read_file(filepath: str) -> str: "pydocstyle", "flake8", "pre-commit", - *torch_requirements, ], } @@ -76,9 +73,6 @@ def read_file(filepath: str) -> str: extras_require=extras_require, test_suite="pytest", platforms=["Linux"], - entry_points={ - "console_scripts": ["smac = smac.smac_cli:cmd_line_call"], - }, classifiers=[ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", diff --git a/smac/__init__.py b/smac/__init__.py index f3fe5805b..ab2781297 100644 --- a/smac/__init__.py +++ b/smac/__init__.py @@ -19,7 +19,7 @@ Copyright {datetime.date.today().strftime('%Y')}, Marius Lindauer, Katharina Eggensperger, Matthias Feurer, André Biedenkapp, Difan Deng, Carolin Benjamins, Tim Ruhkopf, René Sass and Frank Hutter""" -version = "2.0.1" +version = "2.0.2" try: diff --git a/smac/facade/abstract_facade.py b/smac/facade/abstract_facade.py index adb141112..a5db5ab52 100644 --- a/smac/facade/abstract_facade.py +++ b/smac/facade/abstract_facade.py @@ -92,8 +92,10 @@ class AbstractFacade: Callbacks, which are incorporated into the optimization loop. overwrite: bool, defaults to False When True, overwrites the run results if a previous run is found that is - inconsistent in the meta data with the current setup. If ``overwrite`` is set to False, the user is asked - for the exact behaviour (overwrite completely, save old run, or use old results). + consistent in the meta data with the current setup. When False and a previous run is found that is + consistent in the meta data, the run is continued. When False and a previous run is found that is + not consistent in the meta data, the the user is asked for the exact behaviour (overwrite completely + or rename old run first). dask_client: Client | None, defaults to None User-created dask client, which can be used to start a dask cluster and then attach SMAC to it. This will not be closed automatically and will have to be closed manually if provided explicitly. If none is provided @@ -310,6 +312,9 @@ def optimize(self, *, data_to_scatter: dict[str, Any] | None = None) -> Configur Best found configuration. """ incumbents = None + if isinstance(data_to_scatter, dict) and len(data_to_scatter) == 0: + raise ValueError("data_to_scatter must be None or dict with some elements, but got an empty dict.") + try: incumbents = self._optimizer.optimize(data_to_scatter=data_to_scatter) finally: @@ -455,6 +460,18 @@ def _validate(self) -> None: # Make sure the same acquisition function is used assert self._acquisition_function == self._acquisition_maximizer._acquisition_function + if isinstance(self._runner, DaskParallelRunner) and ( + self.scenario.trial_walltime_limit is not None or self.scenario.trial_memory_limit is not None + ): + # This is probably due to pickling dask jobs + raise ValueError( + "Parallelization via Dask cannot be used in combination with limiting " + "the resources " + "of the target function via `scenario.trial_walltime_limit` or " + "`scenario.trial_memory_limit`. Set those to `None` if you want " + "parallelization. " + ) + def _get_signature_arguments(self) -> list[str]: """Returns signature arguments, which are required by the intensifier.""" arguments = [] diff --git a/smac/intensifier/successive_halving.py b/smac/intensifier/successive_halving.py index e7061a190..01b95c6d8 100644 --- a/smac/intensifier/successive_halving.py +++ b/smac/intensifier/successive_halving.py @@ -558,7 +558,8 @@ def _get_best_configs( # If we have more selected configs, we remove the ones with the smallest crowding distance if len(selected_configs) > n_configs: - selected_configs = sort_by_crowding_distance(rh, configs, all_keys)[:n_configs] + all_keys = [from_keys for _ in selected_configs] + selected_configs = sort_by_crowding_distance(rh, selected_configs, all_keys)[:n_configs] logger.debug("Found more configs than required. Removed configs with smallest crowding distance.") return selected_configs diff --git a/smac/main/config_selector.py b/smac/main/config_selector.py index 76ace17f8..7b2053eee 100644 --- a/smac/main/config_selector.py +++ b/smac/main/config_selector.py @@ -266,14 +266,14 @@ def _collect_data(self) -> tuple[np.ndarray, np.ndarray, np.ndarray]: assert self._runhistory_encoder is not None # If we use a float value as a budget, we want to train the model only on the highest budget - available_budgets = [] - for run_key in self._runhistory: - budget = run_key.budget - if budget not in available_budgets: - available_budgets.append(run_key.budget) - - # Sort available budgets from highest to lowest budget - available_budgets = sorted(list(set(available_budgets)), reverse=True) # type: ignore + unique_budgets: set[float] = {run_key.budget for run_key in self._runhistory if run_key.budget is not None} + + available_budgets: list[float] | list[None] + if len(unique_budgets) > 0: + # Sort available budgets from highest to lowest budget + available_budgets = sorted(unique_budgets, reverse=True) + else: + available_budgets = [None] # Get #points per budget and if there are enough samples, then build a model for b in available_budgets: diff --git a/smac/main/smbo.py b/smac/main/smbo.py index 30e01c2e9..6138e4d64 100644 --- a/smac/main/smbo.py +++ b/smac/main/smbo.py @@ -461,18 +461,22 @@ def _add_results(self) -> None: logger.info("Cost threshold was reached. Abort is requested.") self._stop = True - def register_callback(self, callback: Callback, index: int = -1) -> None: + def register_callback(self, callback: Callback, index: int | None = None) -> None: """ Registers a callback to be called before, in between, and after the Bayesian optimization loop. + Callback is appended to the list by default. Parameters ---------- callback : Callback The callback to be registered. - index : int - The index at which the callback should be registered. + index : int, optional + The index at which the callback should be registered. The default is None. + If it is None, append the callback to the list. """ + if index is None: + index = len(self._callbacks) self._callbacks.insert(index, callback) def _initialize_state(self) -> None: @@ -501,7 +505,7 @@ def _initialize_state(self) -> None: logger.info("Since the previous run was not successful, SMAC will start from scratch again.") self.reset() else: - # Here, we run into differen scenarios + # Here, we run into different scenarios diff = recursively_compare_dicts( Scenario.make_serializable(self._scenario), Scenario.make_serializable(old_scenario), diff --git a/smac/model/gaussian_process/kernels/base_kernels.py b/smac/model/gaussian_process/kernels/base_kernels.py index 044ad4fca..4dd6ff6bb 100644 --- a/smac/model/gaussian_process/kernels/base_kernels.py +++ b/smac/model/gaussian_process/kernels/base_kernels.py @@ -250,12 +250,12 @@ def _signature(self, func: Callable) -> Signature: def _set_active_dims(self, operate_on: np.ndarray | None = None) -> None: """Sets dimensions this kernel should work on.""" - if operate_on is not None and type(operate_on) in (list, np.ndarray): + if operate_on is not None and isinstance(operate_on, (list, np.ndarray)): if not isinstance(operate_on, np.ndarray): - raise TypeError("The argument `operate_on` needs to be of type np.ndarray but is %s" % type(operate_on)) + raise TypeError(f"The argument `operate_on` needs to be of type np.ndarray but is {type(operate_on)}") - if operate_on.dtype != int: - raise ValueError("The dtype of argument `operate_on` needs to be int, but is %s" % operate_on.dtype) + if not np.issubdtype(operate_on.dtype, np.integer): + raise ValueError(f"The dtype of `operate_on` needs to be np.integer, but is {operate_on.dtype}") self.operate_on = operate_on self._len_active = len(operate_on) diff --git a/smac/runhistory/runhistory.py b/smac/runhistory/runhistory.py index a30e1356e..aaf88889c 100644 --- a/smac/runhistory/runhistory.py +++ b/smac/runhistory/runhistory.py @@ -435,7 +435,7 @@ def get_min_cost(self, config: Configuration) -> float: cost = self._min_cost_per_config.get(config_id, np.nan) # type: ignore if self._n_objectives > 1: - assert type(cost) == list + assert isinstance(cost, list) assert self.multi_objective_algorithm is not None costs = normalize_costs(cost, self._objective_bounds) @@ -443,7 +443,7 @@ def get_min_cost(self, config: Configuration) -> float: # Note: We have to mean here because we already got the min cost return self.multi_objective_algorithm(costs) - assert type(cost) == float + assert isinstance(cost, float) return float(cost) def average_cost( @@ -847,7 +847,7 @@ def load(self, filename: str | Path, configspace: ConfigurationSpace) -> None: for entry in data["data"]: # Set n_objectives first if self._n_objectives == -1: - if isinstance(entry[4], float) or isinstance(entry[4], int): + if isinstance(entry[4], (float, int)): self._n_objectives = 1 else: self._n_objectives = len(entry[4]) diff --git a/smac/runner/target_function_runner.py b/smac/runner/target_function_runner.py index 28837caa0..5431f5ed4 100644 --- a/smac/runner/target_function_runner.py +++ b/smac/runner/target_function_runner.py @@ -7,6 +7,7 @@ import math import time import traceback +from functools import partial import numpy as np from ConfigSpace import Configuration @@ -88,7 +89,17 @@ def __init__( @property def meta(self) -> dict[str, Any]: # noqa: D102 meta = super().meta - meta.update({"code": str(self._target_function.__code__.co_code)}) + + # Partial's don't have a __code__ attribute but are a convenient + # way a user might want to pass a function to SMAC, specifying + # keyword arguments. + f = self._target_function + if isinstance(f, partial): + f = f.func + meta.update({"code": str(f.__code__.co_code)}) + meta.update({"code-partial-args": repr(f)}) + else: + meta.update({"code": str(self._target_function.__code__.co_code)}) return meta diff --git a/smac/utils/configspace.py b/smac/utils/configspace.py index e6fdeddbd..8f281ed3e 100644 --- a/smac/utils/configspace.py +++ b/smac/utils/configspace.py @@ -169,12 +169,17 @@ def print_config_changes( if incumbent is None or challenger is None: return - params = sorted([(param, incumbent[param], challenger[param]) for param in challenger.keys()]) - for param in params: - if param[1] != param[2]: - logger.info("--- %s: %r -> %r" % param) - else: - logger.debug("--- %s Remains unchanged: %r", param[0], param[1]) + inc_keys = set(incumbent.keys()) + all_keys = inc_keys.union(challenger.keys()) + + lines = [] + for k in sorted(all_keys): + inc_k = incumbent.get(k, "-inactive-") + cha_k = challenger.get(k, "-inactive-") + lines.append(f"--- {k}: {inc_k} -> {cha_k}" + " (unchanged)" if inc_k == cha_k else "") + + msg = "\n".join(lines) + logger.debug(msg) # def check_subspace_points( diff --git a/smac/utils/multi_objective.py b/smac/utils/multi_objective.py index 2dec4da94..f959fe783 100644 --- a/smac/utils/multi_objective.py +++ b/smac/utils/multi_objective.py @@ -29,7 +29,7 @@ def normalize_costs( costs = [] for v, b in zip(values, bounds): - assert type(v) != list + assert not isinstance(v, list) p = v - b[0] q = b[1] - b[0] diff --git a/tests/fixtures/models.py b/tests/fixtures/models.py index 1321176f2..29f9bf152 100644 --- a/tests/fixtures/models.py +++ b/tests/fixtures/models.py @@ -71,7 +71,7 @@ def train(self, config: Configuration, instance: str = "0-1", budget: float = 1, # SGD classifier using given configuration clf = SGDClassifier( - loss="log", + loss="log_loss", penalty="elasticnet", alpha=config["alpha"], l1_ratio=config["l1_ratio"], diff --git a/tests/fixtures/scenario.py b/tests/fixtures/scenario.py index 621193987..4830c68c6 100644 --- a/tests/fixtures/scenario.py +++ b/tests/fixtures/scenario.py @@ -51,7 +51,7 @@ def _make( instance_features=instance_features, min_budget=min_budget, max_budget=max_budget, - use_default_config=use_default_config + use_default_config=use_default_config, ) return _make diff --git a/tests/test_initial_design/test_initial_design.py b/tests/test_initial_design/test_initial_design.py index d12e503ff..ed08db0d5 100644 --- a/tests/test_initial_design/test_initial_design.py +++ b/tests/test_initial_design/test_initial_design.py @@ -122,6 +122,7 @@ def test_select_configurations(make_scenario, configspace_small): with pytest.raises(NotImplementedError): dc.select_configurations() + def test_include_default_config(make_scenario, configspace_small): scenario = make_scenario(configspace_small, use_default_config=True) @@ -133,4 +134,3 @@ def test_include_default_config(make_scenario, configspace_small): # if use_default_config is True, then the default config should be included in the additional_configs default_config = scenario.configspace.get_default_configuration() assert default_config in dc._additional_configs - From a15c1b79f124f9c8533eb494a9e65d90426ee83e Mon Sep 17 00:00:00 2001 From: Difan Deng <33290713+dengdifan@users.noreply.github.com> Date: Thu, 5 Oct 2023 15:54:43 +0200 Subject: [PATCH 14/57] allow rf to impute OrdinalHyperparameter (#1065) * allow rf to impute OrdinalHyperparameter * Add missing period in docs. * Add issue number in changelog. --------- Co-authored-by: Sarah Krebs --- CHANGELOG.md | 5 +++++ smac/model/random_forest/abstract_random_forest.py | 5 ++++- tests/test_model/test_rf.py | 14 +++++++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbcac80e4..6df038b75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# 2.0.3 + +## Bugfixes +- Add OrdinalHyperparameter for random forest imputer (#1065). + # 2.0.2 ## Improvements diff --git a/smac/model/random_forest/abstract_random_forest.py b/smac/model/random_forest/abstract_random_forest.py index d4f1f7ce3..e407331be 100644 --- a/smac/model/random_forest/abstract_random_forest.py +++ b/smac/model/random_forest/abstract_random_forest.py @@ -6,6 +6,7 @@ from ConfigSpace import ( CategoricalHyperparameter, Constant, + OrdinalHyperparameter, UniformFloatHyperparameter, UniformIntegerHyperparameter, ) @@ -36,12 +37,14 @@ def _impute_inactive(self, X: np.ndarray) -> np.ndarray: self._conditional[idx] = True if isinstance(hp, CategoricalHyperparameter): self._impute_values[idx] = len(hp.choices) + elif isinstance(hp, OrdinalHyperparameter): + self._impute_values[idx] = len(hp.sequence) elif isinstance(hp, (UniformFloatHyperparameter, UniformIntegerHyperparameter)): self._impute_values[idx] = -1 elif isinstance(hp, Constant): self._impute_values[idx] = 1 else: - raise ValueError + raise ValueError(f"Unsupported hyperparameter type: {type(hp)}") if self._conditional[idx] is True: nonfinite_mask = ~np.isfinite(X[:, idx]) diff --git a/tests/test_model/test_rf.py b/tests/test_model/test_rf.py index a30995b23..e59b2ded1 100644 --- a/tests/test_model/test_rf.py +++ b/tests/test_model/test_rf.py @@ -257,24 +257,36 @@ def test_with_ordinal(): def test_impute_inactive_hyperparameters(): cs = ConfigurationSpace(seed=0) - a = cs.add_hyperparameter(CategoricalHyperparameter("a", [0, 1])) + a = cs.add_hyperparameter(CategoricalHyperparameter("a", [0, 1, 2])) b = cs.add_hyperparameter(CategoricalHyperparameter("b", [0, 1])) c = cs.add_hyperparameter(UniformFloatHyperparameter("c", 0, 1)) + d = cs.add_hyperparameter(OrdinalHyperparameter("d", [0, 1, 2])) cs.add_condition(EqualsCondition(b, a, 1)) cs.add_condition(EqualsCondition(c, a, 0)) + cs.add_condition(EqualsCondition(d, a, 2)) configs = cs.sample_configuration(size=100) config_array = convert_configurations_to_array(configs) for line in config_array: if line[0] == 0: assert np.isnan(line[1]) + assert np.isnan(line[3]) elif line[0] == 1: assert np.isnan(line[2]) + assert np.isnan(line[3]) + elif line[0] == 2: + assert np.isnan(line[1]) + assert np.isnan(line[2]) model = RandomForest(configspace=cs) config_array = model._impute_inactive(config_array) for line in config_array: if line[0] == 0: assert line[1] == 2 + assert line[3] == 3 elif line[0] == 1: assert line[2] == -1 + assert line[3] == 3 + elif line[0] == 2: + assert line[1] == 2 + assert line[2] == -1 From d58d5b1e74bf107b2a85913ac062b776f13f2cc6 Mon Sep 17 00:00:00 2001 From: Weihuang Wen Date: Thu, 5 Oct 2023 23:40:33 +0800 Subject: [PATCH 15/57] fix dask_scheduler_file path (#1055) * fix dask_scheduler_file path * update change log --------- Co-authored-by: dengdifan --- CHANGELOG.md | 1 + smac/runner/dask_runner.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6df038b75..aa2d5ae48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Bugfixes - Add OrdinalHyperparameter for random forest imputer (#1065). +- Fix path for dask scheduler file (#1055). # 2.0.2 diff --git a/smac/runner/dask_runner.py b/smac/runner/dask_runner.py index b9aade401..d4bb528bd 100644 --- a/smac/runner/dask_runner.py +++ b/smac/runner/dask_runner.py @@ -91,7 +91,7 @@ def __init__( ) if self._scenario.output_directory is not None: - self._scheduler_file = self._scenario.output_directory / ".dask_scheduler_file" + self._scheduler_file = Path(self._scenario.output_directory, ".dask_scheduler_file") self._client.write_scheduler_file(scheduler_file=str(self._scheduler_file)) else: # We just use their set up From 06d584f5b1b9614ba2b900b03bf0ad6ef294a6c7 Mon Sep 17 00:00:00 2001 From: PhilippBordne <71140732+PhilippBordne@users.noreply.github.com> Date: Mon, 9 Oct 2023 11:42:02 +0200 Subject: [PATCH 16/57] Fix config rejection for #1068 (#1069) * racing: make sure config only deleted from rejected configs if among incumbents upon update * test case for incumbent rejection * update code comments * added missing newline ahead of new test case * update change log --------- Co-authored-by: dengdifan Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> --- CHANGELOG.md | 3 +- smac/intensifier/abstract_intensifier.py | 8 +++- .../test_abstract_intensifier.py | 42 +++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa2d5ae48..5ffa77ab5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ # 2.0.3 ## Bugfixes -- Add OrdinalHyperparameter for random forest imputer (#1065). - Fix path for dask scheduler file (#1055). +- Add OrdinalHyperparameter for random forest imputer (#1065). +- Configurations that fail to become incumbents will be added to the rejected lists (#1069). # 2.0.2 diff --git a/smac/intensifier/abstract_intensifier.py b/smac/intensifier/abstract_intensifier.py index b94486727..cb537e9cc 100644 --- a/smac/intensifier/abstract_intensifier.py +++ b/smac/intensifier/abstract_intensifier.py @@ -571,8 +571,12 @@ def update_incumbents(self, config: Configuration) -> None: if len(previous_incumbents) == len(new_incumbents): if previous_incumbents == new_incumbents: - # No changes in the incumbents - self._remove_rejected_config(config_id) + # No changes in the incumbents, we need this clause because we can't use set difference then + if config_id in new_incumbent_ids: + self._remove_rejected_config(config_id) + else: + # config worse than incumbents and thus rejected + self._add_rejected_config(config_id) return else: # In this case, we have to determine which config replaced which incumbent and reject it diff --git a/tests/test_intensifier/test_abstract_intensifier.py b/tests/test_intensifier/test_abstract_intensifier.py index ce980c49b..b8dc91a1c 100644 --- a/tests/test_intensifier/test_abstract_intensifier.py +++ b/tests/test_intensifier/test_abstract_intensifier.py @@ -109,6 +109,48 @@ def test_incumbent_selection_multi_objective(make_scenario, configspace_small, m assert intensifier.get_incumbents() == [config] +def test_config_rejection_single_objective(configspace_small, make_scenario): + """ Tests whether configs are rejected properly if they are worse than the incumbent. """ + scenario = make_scenario(configspace_small, use_instances=False) + runhistory = RunHistory() + intensifier = Intensifier(scenario=scenario) + intensifier.runhistory = runhistory + + configs = configspace_small.sample_configuration(3) + + runhistory.add(config=configs[0], + cost=5, + time=0.0, + seed=0, + status=StatusType.SUCCESS, + force_update=True) + intensifier.update_incumbents(configs[0]) + + assert intensifier._rejected_config_ids == [] + + # add config that yielded better results, updating incumbent and sending prior incumbent to rejected + runhistory.add(config=configs[1], + cost=1, + time=0.0, + seed=0, + status=StatusType.SUCCESS, + force_update=True) + intensifier.update_incumbents(config=configs[1]) + + assert intensifier._rejected_config_ids == [1] + + # add config that is no better should thus go to rejected + runhistory.add(config=configs[2], + cost=1, + time=0.0, + seed=0, + status=StatusType.SUCCESS, + force_update=True) + intensifier.update_incumbents(config=configs[2]) + + assert intensifier._rejected_config_ids == [1, 3] + + def test_incumbent_differences(make_scenario, configspace_small): pass From 3852013bb4bd8ddecbf87b3cd78904f850e8f1b5 Mon Sep 17 00:00:00 2001 From: Eddie Bergman Date: Tue, 21 Nov 2023 14:43:47 +0100 Subject: [PATCH 17/57] fix(RandomForest): Always cast `seed` to `int` (#1084) * fix(RandomForest): Always cast `seed` to `int` In cases where seeds were generated from some numpy objects, sometimes you'd get back an `np.integer`, which causes the program to crash when communicating with `pyrfr` through `swig`. It seems that swig doesn't know that it's an _int-like_ and so we explicitly cast it to `int`. * Update CHANGELOG.md --------- Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> --- CHANGELOG.md | 1 + smac/model/random_forest/random_forest.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ffa77ab5..2e27089d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Fix path for dask scheduler file (#1055). - Add OrdinalHyperparameter for random forest imputer (#1065). - Configurations that fail to become incumbents will be added to the rejected lists (#1069). +- SMAC RandomForest doesn't crash when `np.integer` used, i.e. as generated from a `np.random.RandomState` (#1084). # 2.0.2 diff --git a/smac/model/random_forest/random_forest.py b/smac/model/random_forest/random_forest.py index 6d7840d2a..634e0af06 100644 --- a/smac/model/random_forest/random_forest.py +++ b/smac/model/random_forest/random_forest.py @@ -87,7 +87,9 @@ def __init__( self._rf_opts.compute_law_of_total_variance = False self._rf: BinaryForest | None = None self._log_y = log_y - self._rng = regression.default_random_engine(seed) + + # Case to `int` incase we get an `np.integer` type + self._rng = regression.default_random_engine(int(seed)) self._n_trees = n_trees self._n_points_per_tree = n_points_per_tree From 6280104887189ce0c652e04eb2017468cd8c399d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Nov 2023 15:01:00 +0100 Subject: [PATCH 18/57] Bump actions/checkout from 3 to 4 (#1072) * Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update CHANGELOG.md --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: benjamc --- .github/workflows/citation.yml | 2 +- .github/workflows/dist.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/pre-commit-update.yml | 2 +- .github/workflows/pre-commit.yml | 2 +- .github/workflows/pytest.yml | 6 +++--- CHANGELOG.md | 3 +++ 7 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/citation.yml b/.github/workflows/citation.yml index 9ca4ace4b..91509b900 100644 --- a/.github/workflows/citation.yml +++ b/.github/workflows/citation.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out a copy of the repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Check whether the citation metadata from CITATION.cff is valid uses: citation-file-format/cffconvert-github-action@2.0.0 diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index 0965ea2f1..39f187b0f 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d89791712..d97c162f0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 diff --git a/.github/workflows/pre-commit-update.yml b/.github/workflows/pre-commit-update.yml index 1688f60d7..d475d0a9b 100644 --- a/.github/workflows/pre-commit-update.yml +++ b/.github/workflows/pre-commit-update.yml @@ -11,7 +11,7 @@ jobs: auto-update: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index e9ba1a01b..d22888093 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: recursive diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 37e15a472..8ce02637d 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -54,7 +54,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 @@ -108,7 +108,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Conda install uses: conda-incubator/setup-miniconda@v2 @@ -148,7 +148,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e27089d3..123588564 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ - Configurations that fail to become incumbents will be added to the rejected lists (#1069). - SMAC RandomForest doesn't crash when `np.integer` used, i.e. as generated from a `np.random.RandomState` (#1084). +## Misc +- ci: Update action version (#1072). + # 2.0.2 ## Improvements From 6d46430e780a5e7a4053f31a1b0ebce71e4ccc19 Mon Sep 17 00:00:00 2001 From: SimonThormeyer <49559340+SimonThormeyer@users.noreply.github.com> Date: Mon, 27 Nov 2023 16:31:36 +0100 Subject: [PATCH 19/57] chore: log warn on condition when custom dask client is provided (#1071) --- CHANGELOG.md | 3 +++ smac/facade/abstract_facade.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 123588564..1dee8fed2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ ## Misc - ci: Update action version (#1072). +## Minor +- When a custom dask client is provided, emit the warning that the `n_workers` parameter is ignored only if it deviates from its default value, `1` ([#1071](https://github.com/automl/SMAC3/pull/1071)). + # 2.0.2 ## Improvements diff --git a/smac/facade/abstract_facade.py b/smac/facade/abstract_facade.py index a5db5ab52..c98451a27 100644 --- a/smac/facade/abstract_facade.py +++ b/smac/facade/abstract_facade.py @@ -190,7 +190,7 @@ def __init__( # In case of multiple jobs, we need to wrap the runner again using DaskParallelRunner if (n_workers := scenario.n_workers) > 1 or dask_client is not None: - if dask_client is not None: + if dask_client is not None and n_workers > 1: logger.warning( "Provided `dask_client`. Ignore `scenario.n_workers`, directly set `n_workers` in `dask_client`." ) From 3d09049c48036ca450d04d44bebc42dd48c1a9f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 09:16:01 +0100 Subject: [PATCH 20/57] Bump conda-incubator/setup-miniconda from 2 to 3 (#1087) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 2 to 3. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v2...v3) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 8ce02637d..4c1afcd4d 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -111,7 +111,7 @@ jobs: uses: actions/checkout@v4 - name: Conda install - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true python-version: ${{ matrix.python-version }} From a7ea4b59d0a9c76a2050153df55550b477d03916 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 10 Jan 2024 03:03:31 -0600 Subject: [PATCH 21/57] Propagate the Scenario random seed to get_random_design (#1066) * Propagate the Scenario's random seed to get_random_design * add changelog line --------- Co-authored-by: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> --- CHANGELOG.md | 1 + smac/facade/hyperparameter_optimization_facade.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dee8fed2..dc5a8df29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Bugfixes - Fix path for dask scheduler file (#1055). - Add OrdinalHyperparameter for random forest imputer (#1065). +- Propagate the Scenario random seed to `get_random_design` (#1066). - Configurations that fail to become incumbents will be added to the rejected lists (#1069). - SMAC RandomForest doesn't crash when `np.integer` used, i.e. as generated from a `np.random.RandomState` (#1084). diff --git a/smac/facade/hyperparameter_optimization_facade.py b/smac/facade/hyperparameter_optimization_facade.py index 70a85517c..83207143a 100644 --- a/smac/facade/hyperparameter_optimization_facade.py +++ b/smac/facade/hyperparameter_optimization_facade.py @@ -178,7 +178,7 @@ def get_random_design( # type: ignore probability : float, defaults to 0.2 Probability that a configuration will be drawn at random. """ - return ProbabilityRandomDesign(probability=probability) + return ProbabilityRandomDesign(probability=probability, seed=scenario.seed) @staticmethod def get_multi_objective_algorithm( # type: ignore From c0e77fad499daa1751cd3b8df8295c3f2bd294d3 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Thu, 8 Feb 2024 10:44:46 +0100 Subject: [PATCH 22/57] [#1056] Add example on intensify for cross-validation. (#1061) * Add example on intensify for cross-validation. * Improve example * Address Difan's comment * Update CHANGELOG.md --- CHANGELOG.md | 3 +- .../4_intensify_crossvalidation.py | 127 ++++++++++++++++++ 2 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 examples/4_advanced_optimizer/4_intensify_crossvalidation.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dc5a8df29..1ee1b9922 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,9 +16,10 @@ # 2.0.2 ## Improvements -- Add an error when we get an empty dict data_to_scatter so that we can avoid an internal error caused in Dask precautiously +- Add an error when we get an empty dict data_to_scatter so that we can avoid an internal error caused in Dask precautiously. - Add experimental instruction for installing SMAC in Windows via a WSL. - More detailed documentation regarding continuing runs. +- Add a new example that demonstrates the use of intensification to speed up cross-validation for machine learning. ## Bugfixes - Fix bug in the incumbent selection in the case that multi-fidelity is combined with multi-objective (#1019). diff --git a/examples/4_advanced_optimizer/4_intensify_crossvalidation.py b/examples/4_advanced_optimizer/4_intensify_crossvalidation.py new file mode 100644 index 000000000..d215dd8ec --- /dev/null +++ b/examples/4_advanced_optimizer/4_intensify_crossvalidation.py @@ -0,0 +1,127 @@ +""" +Speeding up Cross-Validation with Intensification +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +An example of optimizing a simple support vector machine on the digits dataset. In contrast to the +[simple example](examples/1_basics/2_svm_cv.py), in which all cross-validation folds are executed +at once, we use the intensification mechanism described in the original +[SMAC paper](https://link.springer.com/chapter/10.1007/978-3-642-25566-3_40) as also demonstrated +by [Auto-WEKA](https://dl.acm.org/doi/10.1145/2487575.2487629). This mechanism allows us to +terminate the evaluation of a configuration if after a certain number of folds, the configuration +is found to be worse than the incumbent configuration. This is especially useful if the evaluation +of a configuration is expensive, e.g., if we have to train a neural network or if we have to +evaluate the configuration on a large dataset. +""" +__copyright__ = "Copyright 2023, AutoML.org Freiburg-Hannover" +__license__ = "3-clause BSD" + +N_FOLDS = 10 # Global variable that determines the number of folds + +from ConfigSpace import Configuration, ConfigurationSpace, Float +from sklearn import datasets, svm +from sklearn.model_selection import StratifiedKFold + +from smac import HyperparameterOptimizationFacade, Scenario +from smac.intensifier import Intensifier + +# We load the digits dataset, a small-scale 10-class digit recognition dataset +X, y = datasets.load_digits(return_X_y=True) + + +class SVM: + @property + def configspace(self) -> ConfigurationSpace: + # Build Configuration Space which defines all parameters and their ranges + cs = ConfigurationSpace(seed=0) + + # First we create our hyperparameters + C = Float("C", (2 ** - 5, 2 ** 15), default=1.0, log=True) + gamma = Float("gamma", (2 ** -15, 2 ** 3), default=1.0, log=True) + + # Add hyperparameters to our configspace + cs.add_hyperparameters([C, gamma]) + + return cs + + def train(self, config: Configuration, instance: str, seed: int = 0) -> float: + """Creates a SVM based on a configuration and evaluate on the given fold of the digits dataset + + Parameters + ---------- + config: Configuration + The configuration to train the SVM. + instance: str + The name of the instance this configuration should be evaluated on. This is always of type + string by definition. In our case we cast to int, but this could also be the filename of a + problem instance to be loaded. + seed: int + The seed used for this call. + """ + instance = int(instance) + classifier = svm.SVC(**config, random_state=seed) + splitter = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=seed) + for k, (train_idx, test_idx) in enumerate(splitter.split(X=X, y=y)): + if k != instance: + continue + else: + train_X = X[train_idx] + train_y = y[train_idx] + test_X = X[test_idx] + test_y = y[test_idx] + classifier.fit(train_X, train_y) + cost = 1 - classifier.score(test_X, test_y) + + return cost + + +if __name__ == "__main__": + classifier = SVM() + + # Next, we create an object, holding general information about the run + scenario = Scenario( + classifier.configspace, + n_trials=50, # We want to run max 50 trials (combination of config and instances in the case of + # deterministic=True. In the case of deterministic=False, this would be the + # combination of instances, seeds and configs). The number of distinct configurations + # evaluated by SMAC will be lower than this number because some of the configurations + # will be executed on more than one instance (CV fold). + instances=[f"{i}" for i in range(N_FOLDS)], # Specify all instances by their name (as a string) + instance_features={f"{i}": [i] for i in range(N_FOLDS)}, # breaks SMAC + deterministic=True # To simplify the problem we make SMAC believe that we have a deterministic + # optimization problem. + + ) + + # We want to run the facade's default initial design, but we want to change the number + # of initial configs to 5. + initial_design = HyperparameterOptimizationFacade.get_initial_design(scenario, n_configs=5) + + # Now we use SMAC to find the best hyperparameters + smac = HyperparameterOptimizationFacade( + scenario, + classifier.train, + initial_design=initial_design, + overwrite=True, # If the run exists, we overwrite it; alternatively, we can continue from last state + # The next line defines the intensifier, i.e., the module that governs the selection of + # instance-seed pairs. Since we set deterministic to True above, it only governs the instance in + # this example. Technically, it is not necessary to create the intensifier as a user, but it is + # necessary to do so because we change the argument max_config_calls (the number of instance-seed pairs + # per configuration to try) to the number of cross-validation folds, while the default would be 3. + intensifier=Intensifier(scenario=scenario, max_config_calls=N_FOLDS, seed=0) + ) + + incumbent = smac.optimize() + + # Get cost of default configuration + default_cost = smac.validate(classifier.configspace.get_default_configuration()) + print(f"Default cost: {default_cost}") + + # Let's calculate the cost of the incumbent + incumbent_cost = smac.validate(incumbent) + print(f"Incumbent cost: {incumbent_cost}") + + # Let's see how many configurations we have evaluated. If this number is higher than 5, we have looked + # at more configurations than would have been possible with regular cross-validation, where the number + # of configurations would be determined by the number of trials divided by the number of folds (50 / 10). + runhistory = smac.runhistory + print(f"Number of evaluated configurations: {len(runhistory.config_ids)}") \ No newline at end of file From 29b8a01f0460d5fdeb3ff597cc51300797b2643d Mon Sep 17 00:00:00 2001 From: helegraf Date: Tue, 13 Feb 2024 13:45:56 +0100 Subject: [PATCH 23/57] Change stale-reminder to recent-reminder (#1096) --- .github/stale.yml | 30 ----------- .github/workflows/recent_reminder.yml | 46 ++++++++++++++++ .github/workflows/stale_reminder.yml | 77 --------------------------- 3 files changed, 46 insertions(+), 107 deletions(-) delete mode 100644 .github/stale.yml create mode 100644 .github/workflows/recent_reminder.yml delete mode 100644 .github/workflows/stale_reminder.yml diff --git a/.github/stale.yml b/.github/stale.yml deleted file mode 100644 index 5f6a3d7b5..000000000 --- a/.github/stale.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Number of days of inactivity before an issue becomes stale -daysUntilStale: 60 - -# Number of days of inactivity before a stale issue is closed -daysUntilClose: 14 - -# Issues with these labels will never be considered stale -exemptLabels: - - bug - - dependency - - documentation - - enhancement - - feature - - test - - example - - discussion - - duplicate - - question - -# Label to use when marking an issue as stale -staleLabel: stale - -# Comment to post when marking an issue as stale. Set to `false` to disable -markComment: > - This issue has been automatically marked as stale because it has not had - recent activity. It will be closed if no further activity occurs. Thank you - for your contributions. - -# Comment to post when closing a stale issue. Set to `false` to disable -closeComment: false \ No newline at end of file diff --git a/.github/workflows/recent_reminder.yml b/.github/workflows/recent_reminder.yml new file mode 100644 index 000000000..80fa5fa14 --- /dev/null +++ b/.github/workflows/recent_reminder.yml @@ -0,0 +1,46 @@ +name: Recent-Reminder + +on: + schedule: + - cron: '00 13 * * 1' + workflow_dispatch: + +jobs: + stale-reminder: + runs-on: ubuntu-latest + steps: + - name: Get cutoff dates + id: date + run: | + echo "RECENT_ISSUE_CUTOFF_DATE=$(date -d '-7 days' '+%Y-%m-%d')" >> $GITHUB_ENV + - name: Get list of issues that have had interactions in the last week + id: recent + uses: lee-dohm/select-matching-issues@v1 + with: + format: list + path: "recent_issues.md" + token: ${{ github.token }} + query: >- + is:issue + is:open + updated:>=${{ env.RECENT_ISSUE_CUTOFF_DATE }} + sort:updated-asc + - name: Combine issues into mail content + id: combine + run: | + echo "## Issues that have had interaction in the last 7 days
" >> mail.html + echo "$(" >> mail.html + - name: Send mail + id: mail + uses: dawidd6/action-send-mail@v3 + with: + server_address: ${{secrets.MAIL_SERVER_ADDRESS}} + server_port: ${{secrets.MAIL_SERVER_PORT}} + secure: true + username: ${{secrets.MAIL_USERNAME}} + password: ${{secrets.MAIL_PASSWORD}} + subject: '[Current SMAC3 Issues] Issues that have been interacted with since ${{ env.RECENT_ISSUE_CUTOFF_DATE }}' + to: ${{secrets.MAIL_TARGET}} + from: SMAC3 Stale-Bot <${{secrets.MAIL_ADDRESS}}> + html_body: file://mail.html + convert_markdown: true \ No newline at end of file diff --git a/.github/workflows/stale_reminder.yml b/.github/workflows/stale_reminder.yml deleted file mode 100644 index ce1327683..000000000 --- a/.github/workflows/stale_reminder.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Stale-Reminder - -on: - schedule: - - cron: '00 13 * * 1' - workflow_dispatch: - -jobs: - stale-reminder: - runs-on: ubuntu-latest - steps: - - name: Get cutoff dates - id: date - run: | - echo "CUTOFF_DATE=$(date -d '-46 days' '+%Y-%m-%d')" >> $GITHUB_ENV - echo "RECENT_ISSUE_CUTOFF_DATE=$(date -d '-7 days' '+%Y-%m-%d')" >> $GITHUB_ENV - - name: Get list of issues that have had interactions in the last week - id: recent - uses: lee-dohm/select-matching-issues@v1 - with: - format: list - path: "recent_issues.md" - token: ${{ github.token }} - query: >- - is:issue - is:open - updated:>=${{ env.RECENT_ISSUE_CUTOFF_DATE }} - sort:updated-asc - - name: Collect issues that may become stale - id: stale - uses: lee-dohm/select-matching-issues@v1 - with: - format: list - path: "potentially_stale_issues.md" - token: ${{ github.token }} - query: >- - is:issue - is:open - -label:dependency,documentation,feature,enhancement,bug,test,example,discussion,duplicate,question - updated:<${{ env.CUTOFF_DATE }} - sort:updated-asc - - name: Collect labelled issues that have not had interaction in a long time (but will not become stale) - id: old - uses: lee-dohm/select-matching-issues@v1 - with: - format: list - path: "old_issues.md" - token: ${{ github.token }} - query: >- - is:issue - is:open - label:dependency,documentation,feature,enhancement,bug,test,example,discussion,duplicate,question - updated:<${{ env.CUTOFF_DATE }} - sort:updated-asc - - name: Combine issues into mail content - id: combine - run: | - echo "## Issues that have had interaction in the last 7 days
" >> mail.html - echo "$(" >> mail.html - echo "## Issues that may become stale in <= 14 days
" >> mail.html - echo "$(" >> mail.html - echo "## Issues that have not had interaction in the last 46 days but will not go stale due to their labels
" >> mail.html - echo "$(" >> mail.html - - name: Send mail - id: mail - uses: dawidd6/action-send-mail@v3 - with: - server_address: ${{secrets.MAIL_SERVER_ADDRESS}} - server_port: ${{secrets.MAIL_SERVER_PORT}} - secure: true - username: ${{secrets.MAIL_USERNAME}} - password: ${{secrets.MAIL_PASSWORD}} - subject: '[Stale Issues] Issues with last interaction before ${{ env.CUTOFF_DATE }}' - to: ${{secrets.MAIL_TARGET}} - from: SMAC3 Stale-Bot <${{secrets.MAIL_ADDRESS}}> - html_body: file://mail.html - convert_markdown: true \ No newline at end of file From 9aaa8e94a5b3a9657737a87b903ee96c683cc42c Mon Sep 17 00:00:00 2001 From: helegraf Date: Tue, 13 Feb 2024 13:59:29 +0100 Subject: [PATCH 24/57] Change stale-reminder to recent-reminder (#1095) --- .github/stale.yml | 30 ----------- .github/workflows/recent_reminder.yml | 46 ++++++++++++++++ .github/workflows/stale_reminder.yml | 77 --------------------------- 3 files changed, 46 insertions(+), 107 deletions(-) delete mode 100644 .github/stale.yml create mode 100644 .github/workflows/recent_reminder.yml delete mode 100644 .github/workflows/stale_reminder.yml diff --git a/.github/stale.yml b/.github/stale.yml deleted file mode 100644 index 5f6a3d7b5..000000000 --- a/.github/stale.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Number of days of inactivity before an issue becomes stale -daysUntilStale: 60 - -# Number of days of inactivity before a stale issue is closed -daysUntilClose: 14 - -# Issues with these labels will never be considered stale -exemptLabels: - - bug - - dependency - - documentation - - enhancement - - feature - - test - - example - - discussion - - duplicate - - question - -# Label to use when marking an issue as stale -staleLabel: stale - -# Comment to post when marking an issue as stale. Set to `false` to disable -markComment: > - This issue has been automatically marked as stale because it has not had - recent activity. It will be closed if no further activity occurs. Thank you - for your contributions. - -# Comment to post when closing a stale issue. Set to `false` to disable -closeComment: false \ No newline at end of file diff --git a/.github/workflows/recent_reminder.yml b/.github/workflows/recent_reminder.yml new file mode 100644 index 000000000..80fa5fa14 --- /dev/null +++ b/.github/workflows/recent_reminder.yml @@ -0,0 +1,46 @@ +name: Recent-Reminder + +on: + schedule: + - cron: '00 13 * * 1' + workflow_dispatch: + +jobs: + stale-reminder: + runs-on: ubuntu-latest + steps: + - name: Get cutoff dates + id: date + run: | + echo "RECENT_ISSUE_CUTOFF_DATE=$(date -d '-7 days' '+%Y-%m-%d')" >> $GITHUB_ENV + - name: Get list of issues that have had interactions in the last week + id: recent + uses: lee-dohm/select-matching-issues@v1 + with: + format: list + path: "recent_issues.md" + token: ${{ github.token }} + query: >- + is:issue + is:open + updated:>=${{ env.RECENT_ISSUE_CUTOFF_DATE }} + sort:updated-asc + - name: Combine issues into mail content + id: combine + run: | + echo "## Issues that have had interaction in the last 7 days
" >> mail.html + echo "$(" >> mail.html + - name: Send mail + id: mail + uses: dawidd6/action-send-mail@v3 + with: + server_address: ${{secrets.MAIL_SERVER_ADDRESS}} + server_port: ${{secrets.MAIL_SERVER_PORT}} + secure: true + username: ${{secrets.MAIL_USERNAME}} + password: ${{secrets.MAIL_PASSWORD}} + subject: '[Current SMAC3 Issues] Issues that have been interacted with since ${{ env.RECENT_ISSUE_CUTOFF_DATE }}' + to: ${{secrets.MAIL_TARGET}} + from: SMAC3 Stale-Bot <${{secrets.MAIL_ADDRESS}}> + html_body: file://mail.html + convert_markdown: true \ No newline at end of file diff --git a/.github/workflows/stale_reminder.yml b/.github/workflows/stale_reminder.yml deleted file mode 100644 index ce1327683..000000000 --- a/.github/workflows/stale_reminder.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Stale-Reminder - -on: - schedule: - - cron: '00 13 * * 1' - workflow_dispatch: - -jobs: - stale-reminder: - runs-on: ubuntu-latest - steps: - - name: Get cutoff dates - id: date - run: | - echo "CUTOFF_DATE=$(date -d '-46 days' '+%Y-%m-%d')" >> $GITHUB_ENV - echo "RECENT_ISSUE_CUTOFF_DATE=$(date -d '-7 days' '+%Y-%m-%d')" >> $GITHUB_ENV - - name: Get list of issues that have had interactions in the last week - id: recent - uses: lee-dohm/select-matching-issues@v1 - with: - format: list - path: "recent_issues.md" - token: ${{ github.token }} - query: >- - is:issue - is:open - updated:>=${{ env.RECENT_ISSUE_CUTOFF_DATE }} - sort:updated-asc - - name: Collect issues that may become stale - id: stale - uses: lee-dohm/select-matching-issues@v1 - with: - format: list - path: "potentially_stale_issues.md" - token: ${{ github.token }} - query: >- - is:issue - is:open - -label:dependency,documentation,feature,enhancement,bug,test,example,discussion,duplicate,question - updated:<${{ env.CUTOFF_DATE }} - sort:updated-asc - - name: Collect labelled issues that have not had interaction in a long time (but will not become stale) - id: old - uses: lee-dohm/select-matching-issues@v1 - with: - format: list - path: "old_issues.md" - token: ${{ github.token }} - query: >- - is:issue - is:open - label:dependency,documentation,feature,enhancement,bug,test,example,discussion,duplicate,question - updated:<${{ env.CUTOFF_DATE }} - sort:updated-asc - - name: Combine issues into mail content - id: combine - run: | - echo "## Issues that have had interaction in the last 7 days
" >> mail.html - echo "$(" >> mail.html - echo "## Issues that may become stale in <= 14 days
" >> mail.html - echo "$(" >> mail.html - echo "## Issues that have not had interaction in the last 46 days but will not go stale due to their labels
" >> mail.html - echo "$(" >> mail.html - - name: Send mail - id: mail - uses: dawidd6/action-send-mail@v3 - with: - server_address: ${{secrets.MAIL_SERVER_ADDRESS}} - server_port: ${{secrets.MAIL_SERVER_PORT}} - secure: true - username: ${{secrets.MAIL_USERNAME}} - password: ${{secrets.MAIL_PASSWORD}} - subject: '[Stale Issues] Issues with last interaction before ${{ env.CUTOFF_DATE }}' - to: ${{secrets.MAIL_TARGET}} - from: SMAC3 Stale-Bot <${{secrets.MAIL_ADDRESS}}> - html_body: file://mail.html - convert_markdown: true \ No newline at end of file From a2170f56fe96cd422942f1d2c29eb44ac8506c89 Mon Sep 17 00:00:00 2001 From: helegraf Date: Tue, 5 Mar 2024 14:54:17 +0100 Subject: [PATCH 25/57] Change stale-reminder to recent-reminder (#1095) (#1099) From e1a525b571ab74044675d104a7ba7b3b204c9a33 Mon Sep 17 00:00:00 2001 From: helegraf Date: Tue, 12 Mar 2024 11:07:52 +0100 Subject: [PATCH 26/57] Dont use mutable lists as default args (#1097) * Don't use mutable default argument. If this isn't respected, then additional_configs can be built up overtime when multiple optimizers are created in a single process. This can occur when the use_default_config option is set True and/or when additional_configs are provided. When different ConfigSpaces are provided this can result in an error due to mismatched defaults, for instance. In other cases it can result in non-deterministic behavior due to the random ordering of the configs that get added (e.g., with different parallel pytest runs). - [ ] Cleanup other sources of this error as well elsewhere in the code. - [ ] Add pylint checks for this issue. * fixup some others mutable defaults * add pylint checks * fix more related pylint issues * add changelog line * Update pre-commit config * Update pre-commit config * Remove TODO formulation as an issue already exists for this --------- Co-authored-by: Brian Kroth --- .github/workflows/pre-commit.yml | 1 + .pre-commit-config.yaml | 13 +++++++++++++ CHANGELOG.md | 1 + pyproject.toml | 5 +++++ setup.py | 1 + smac/facade/abstract_facade.py | 5 ++++- smac/facade/algorithm_configuration_facade.py | 4 +++- smac/facade/blackbox_facade.py | 4 +++- .../facade/hyperparameter_optimization_facade.py | 2 +- smac/facade/multi_fidelity_facade.py | 4 +++- smac/facade/random_facade.py | 4 +++- smac/main/config_selector.py | 6 +++--- smac/runhistory/encoder/abstract_encoder.py | 16 ++++++++-------- smac/runhistory/runhistory.py | 4 +++- smac/runner/abstract_runner.py | 4 +++- smac/runner/target_function_runner.py | 4 +++- smac/runner/target_function_script_runner.py | 4 +++- 17 files changed, 61 insertions(+), 21 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index d22888093..e120f1949 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -40,6 +40,7 @@ jobs: run: | pip install pre-commit pre-commit install + pip install pylint - name: Run pre-commit run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dda7e5cae..6b2d3934f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,3 +43,16 @@ repos: name: flake8 smac files: smac exclude: "scripts|tests" + + - repo: local + hooks: + - id: pylint + name: pylint + entry: pylint + language: system + types: [python] + args: + [ + "-rn", # Only display messages + "-sn", # Don't display the score + ] diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ee1b9922..3d3e5e158 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Bugfixes - Fix path for dask scheduler file (#1055). - Add OrdinalHyperparameter for random forest imputer (#1065). +- Don't use mutable default argument (#1067). - Propagate the Scenario random seed to `get_random_design` (#1066). - Configurations that fail to become incumbents will be added to the rejected lists (#1069). - SMAC RandomForest doesn't crash when `np.integer` used, i.e. as generated from a `np.random.RandomState` (#1084). diff --git a/pyproject.toml b/pyproject.toml index c0e1ae4ed..77783d8a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,11 @@ add-ignore = [ # http://www.pydocstyle.org/en/stable/error_codes.html "D415", # First line should end with a period, question mark, or exclamation point ] +[tool.pylint."messages control"] +# FIXME: This is to do a staged introduction of pylint checks for just a single class of problems initially (#1067). +disable = ["all"] +enable = ["dangerous-default-value"] + [tool.mypy] python_version = "3.9" show_error_codes = true diff --git a/setup.py b/setup.py index f194d3677..27ec7e695 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ def read_file(filepath: str) -> str: "pydocstyle", "flake8", "pre-commit", + "pylint", ], } diff --git a/smac/facade/abstract_facade.py b/smac/facade/abstract_facade.py index c98451a27..6ca49b057 100644 --- a/smac/facade/abstract_facade.py +++ b/smac/facade/abstract_facade.py @@ -117,12 +117,15 @@ def __init__( runhistory_encoder: AbstractRunHistoryEncoder | None = None, config_selector: ConfigSelector | None = None, logging_level: int | Path | Literal[False] | None = None, - callbacks: list[Callback] = [], + callbacks: list[Callback] = None, overwrite: bool = False, dask_client: Client | None = None, ): setup_logging(logging_level) + if callbacks is None: + callbacks = [] + if model is None: model = self.get_model(scenario) diff --git a/smac/facade/algorithm_configuration_facade.py b/smac/facade/algorithm_configuration_facade.py index a82e2f92c..58229f4b7 100644 --- a/smac/facade/algorithm_configuration_facade.py +++ b/smac/facade/algorithm_configuration_facade.py @@ -125,7 +125,7 @@ def get_intensifier( def get_initial_design( # type: ignore scenario: Scenario, *, - additional_configs: list[Configuration] = [], + additional_configs: list[Configuration] = None, ) -> DefaultInitialDesign: """Returns an initial design, which returns the default configuration. @@ -134,6 +134,8 @@ def get_initial_design( # type: ignore additional_configs: list[Configuration], defaults to [] Adds additional configurations to the initial design. """ + if additional_configs is None: + additional_configs = [] return DefaultInitialDesign( scenario=scenario, additional_configs=additional_configs, diff --git a/smac/facade/blackbox_facade.py b/smac/facade/blackbox_facade.py index 9e2d4b2d1..33837d127 100644 --- a/smac/facade/blackbox_facade.py +++ b/smac/facade/blackbox_facade.py @@ -240,7 +240,7 @@ def get_initial_design( # type: ignore n_configs: int | None = None, n_configs_per_hyperparamter: int = 8, max_ratio: float = 0.25, - additional_configs: list[Configuration] = [], + additional_configs: list[Configuration] = None, ) -> SobolInitialDesign: """Returns a Sobol design instance. @@ -259,6 +259,8 @@ def get_initial_design( # type: ignore additional_configs: list[Configuration], defaults to [] Adds additional configurations to the initial design. """ + if additional_configs is None: + additional_configs = [] return SobolInitialDesign( scenario=scenario, n_configs=n_configs, diff --git a/smac/facade/hyperparameter_optimization_facade.py b/smac/facade/hyperparameter_optimization_facade.py index 83207143a..3ed09cfde 100644 --- a/smac/facade/hyperparameter_optimization_facade.py +++ b/smac/facade/hyperparameter_optimization_facade.py @@ -138,7 +138,7 @@ def get_initial_design( # type: ignore n_configs: int | None = None, n_configs_per_hyperparamter: int = 10, max_ratio: float = 0.25, - additional_configs: list[Configuration] = [], + additional_configs: list[Configuration] | None = None, ) -> SobolInitialDesign: """Returns a Sobol design instance. diff --git a/smac/facade/multi_fidelity_facade.py b/smac/facade/multi_fidelity_facade.py index fdd52d8a4..1e46b4170 100644 --- a/smac/facade/multi_fidelity_facade.py +++ b/smac/facade/multi_fidelity_facade.py @@ -67,7 +67,7 @@ def get_initial_design( # type: ignore n_configs: int | None = None, n_configs_per_hyperparamter: int = 10, max_ratio: float = 0.25, - additional_configs: list[Configuration] = [], + additional_configs: list[Configuration] = None, ) -> RandomInitialDesign: """Returns a random initial design. @@ -86,6 +86,8 @@ def get_initial_design( # type: ignore additional_configs: list[Configuration], defaults to [] Adds additional configurations to the initial design. """ + if additional_configs is None: + additional_configs = [] return RandomInitialDesign( scenario=scenario, n_configs=n_configs, diff --git a/smac/facade/random_facade.py b/smac/facade/random_facade.py index d5e269534..78038ff2e 100644 --- a/smac/facade/random_facade.py +++ b/smac/facade/random_facade.py @@ -92,7 +92,7 @@ def get_intensifier( def get_initial_design( scenario: Scenario, *, - additional_configs: list[Configuration] = [], + additional_configs: list[Configuration] = None, ) -> DefaultInitialDesign: """Returns an initial design, which returns the default configuration. @@ -101,6 +101,8 @@ def get_initial_design( additional_configs: list[Configuration], defaults to [] Adds additional configurations to the initial design. """ + if additional_configs is None: + additional_configs = [] return DefaultInitialDesign( scenario=scenario, additional_configs=additional_configs, diff --git a/smac/main/config_selector.py b/smac/main/config_selector.py index 7b2053eee..ea57bf0bf 100644 --- a/smac/main/config_selector.py +++ b/smac/main/config_selector.py @@ -91,7 +91,7 @@ def _set_components( acquisition_maximizer: AbstractAcquisitionMaximizer, acquisition_function: AbstractAcquisitionFunction, random_design: AbstractRandomDesign, - callbacks: list[Callback] = [], + callbacks: list[Callback] = None, ) -> None: self._runhistory = runhistory self._runhistory_encoder = runhistory_encoder @@ -99,7 +99,7 @@ def _set_components( self._acquisition_maximizer = acquisition_maximizer self._acquisition_function = acquisition_function self._random_design = random_design - self._callbacks = callbacks + self._callbacks = callbacks if callbacks is not None else [] self._initial_design_configs = initial_design.select_configurations() if len(self._initial_design_configs) == 0: @@ -282,7 +282,7 @@ def _collect_data(self) -> tuple[np.ndarray, np.ndarray, np.ndarray]: if X.shape[0] >= self._min_trials: self._considered_budgets = [b] - # TODO: Add running configs + # Possible add running configs? configs_array = self._runhistory_encoder.get_configurations(budget_subset=self._considered_budgets) return X, Y, configs_array diff --git a/smac/runhistory/encoder/abstract_encoder.py b/smac/runhistory/encoder/abstract_encoder.py index 42454771e..9a891229a 100644 --- a/smac/runhistory/encoder/abstract_encoder.py +++ b/smac/runhistory/encoder/abstract_encoder.py @@ -42,17 +42,17 @@ class AbstractRunHistoryEncoder: def __init__( self, scenario: Scenario, - considered_states: list[StatusType] = [ - StatusType.SUCCESS, - StatusType.CRASHED, - StatusType.MEMORYOUT, - ], - lower_budget_states: list[StatusType] = [], + considered_states: list[StatusType] = None, + lower_budget_states: list[StatusType] = None, scale_percentage: int = 5, seed: int | None = None, ) -> None: if considered_states is None: - raise TypeError("No success states are given.") + considered_states = [ + StatusType.SUCCESS, + StatusType.CRASHED, + StatusType.MEMORYOUT, + ] if seed is None: seed = scenario.seed @@ -62,7 +62,7 @@ def __init__( self._scale_percentage = scale_percentage self._n_objectives = scenario.count_objectives() self._algorithm_walltime_limit = scenario.trial_walltime_limit - self._lower_budget_states = lower_budget_states + self._lower_budget_states = lower_budget_states if lower_budget_states is not None else [] self._considered_states = considered_states self._instances = scenario.instances diff --git a/smac/runhistory/runhistory.py b/smac/runhistory/runhistory.py index aaf88889c..c71384107 100644 --- a/smac/runhistory/runhistory.py +++ b/smac/runhistory/runhistory.py @@ -178,7 +178,7 @@ def add( budget: float | None = None, starttime: float = 0.0, endtime: float = 0.0, - additional_info: dict[str, Any] = {}, + additional_info: dict[str, Any] = None, force_update: bool = False, ) -> None: """Adds a new trial to the RunHistory. @@ -205,6 +205,8 @@ def add( raise TypeError("Configuration must not be None.") elif not isinstance(config, Configuration): raise TypeError("Configuration is not of type Configuration, but %s." % type(config)) + if additional_info is None: + additional_info = {} # Squeeze is important to reduce arrays with one element # to scalars. diff --git a/smac/runner/abstract_runner.py b/smac/runner/abstract_runner.py index 2631bd966..4e15cf4b5 100644 --- a/smac/runner/abstract_runner.py +++ b/smac/runner/abstract_runner.py @@ -50,8 +50,10 @@ class AbstractRunner(ABC): def __init__( self, scenario: Scenario, - required_arguments: list[str] = [], + required_arguments: list[str] = None, ): + if required_arguments is None: + required_arguments = [] self._scenario = scenario self._required_arguments = required_arguments diff --git a/smac/runner/target_function_runner.py b/smac/runner/target_function_runner.py index 5431f5ed4..8a14c2fc9 100644 --- a/smac/runner/target_function_runner.py +++ b/smac/runner/target_function_runner.py @@ -45,8 +45,10 @@ def __init__( self, scenario: Scenario, target_function: Callable, - required_arguments: list[str] = [], + required_arguments: list[str] = None, ): + if required_arguments is None: + required_arguments = [] super().__init__(scenario=scenario, required_arguments=required_arguments) self._target_function = target_function diff --git a/smac/runner/target_function_script_runner.py b/smac/runner/target_function_script_runner.py index 17feffc98..5ab558906 100644 --- a/smac/runner/target_function_script_runner.py +++ b/smac/runner/target_function_script_runner.py @@ -51,8 +51,10 @@ def __init__( self, target_function: str, scenario: Scenario, - required_arguments: list[str] = [], + required_arguments: list[str] = None, ): + if required_arguments is None: + required_arguments = [] super().__init__(scenario=scenario, required_arguments=required_arguments) self._target_function = target_function From e808a00eaa7d043fa73eae3f736dbf98c0d49825 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Mar 2024 11:09:11 +0100 Subject: [PATCH 27/57] Bump actions/setup-python from 4 to 5 (#1089) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dist.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/pre-commit-update.yml | 2 +- .github/workflows/pre-commit.yml | 2 +- .github/workflows/pytest.yml | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index 39f187b0f..8e9908060 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -31,7 +31,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d97c162f0..5f03e854f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -35,7 +35,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" diff --git a/.github/workflows/pre-commit-update.yml b/.github/workflows/pre-commit-update.yml index d475d0a9b..6a2c2f73f 100644 --- a/.github/workflows/pre-commit-update.yml +++ b/.github/workflows/pre-commit-update.yml @@ -13,7 +13,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 - uses: browniebroke/pre-commit-autoupdate-action@main diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index e120f1949..31d957762 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -32,7 +32,7 @@ jobs: submodules: recursive - name: Setup Python 3.10 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 4c1afcd4d..c94e246bb 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -57,7 +57,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -151,7 +151,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} From 658a6b3489bda00797cbe629062ce4e5a651bcad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Mar 2024 11:09:38 +0100 Subject: [PATCH 28/57] Bump peter-evans/create-pull-request from 5 to 6 (#1093) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 5 to 6. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v5...v6) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pre-commit-update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-commit-update.yml b/.github/workflows/pre-commit-update.yml index 6a2c2f73f..b027befb2 100644 --- a/.github/workflows/pre-commit-update.yml +++ b/.github/workflows/pre-commit-update.yml @@ -21,7 +21,7 @@ jobs: run: | pre-commit run --all-files - - uses: peter-evans/create-pull-request@v5 + - uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.GITHUB_TOKEN }} branch: update/pre-commit-hooks From 6da055b26c44fbe619283c60a79ddc8ba0c79161 Mon Sep 17 00:00:00 2001 From: Eddie Bergman Date: Wed, 17 Apr 2024 08:14:37 +0200 Subject: [PATCH 29/57] test: Fix upper bounds for `test_transformer()` and no random in `test_transformer_conditional()` (#1102) * test: Don't rely on randomization for the test * test: Include import * test: Fix boundaries for catgoricals in test * test: Missing import * doc: Make todo not more clear about when to upgrade --- .../test_runhistory_encoder.py | 138 ++++++++++++++---- 1 file changed, 109 insertions(+), 29 deletions(-) diff --git a/tests/test_runhistory/test_runhistory_encoder.py b/tests/test_runhistory/test_runhistory_encoder.py index 7b30bb76a..e3a8f5730 100644 --- a/tests/test_runhistory/test_runhistory_encoder.py +++ b/tests/test_runhistory/test_runhistory_encoder.py @@ -13,6 +13,9 @@ from smac.runhistory.encoder.encoder import RunHistoryEncoder from smac.runner.abstract_runner import StatusType +from ConfigSpace import Configuration +from ConfigSpace.hyperparameters import CategoricalHyperparameter + @pytest.fixture def configs(configspace_small): @@ -39,70 +42,127 @@ def test_transform(runhistory, make_scenario, configspace_small, configs): ) # Normal encoder - encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory + + # TODO: Please replace with the more general solution once ConfigSpace 1.0 + # upper = np.array([hp.upper_vectorized for hp in space.values()]) + # lower = np.array([hp.lower_vectorized for hp in space.values()]) + # - + # Categoricals are upperbounded by their size, rest of hyperparameters are + # upperbounded by 1. + upper_bounds = { + hp.name: (hp.get_size() - 1) + if isinstance(hp, CategoricalHyperparameter) + else 1.0 + for hp in configspace_small.get_hyperparameters() + } + # Need to ensure they match the order in the Configuration vectorized form + sorted_by_indices = sorted( + upper_bounds.items(), + key=lambda x: configspace_small._hyperparameter_idx[x[0]], + ) + upper = np.array([upper_bound for _, upper_bound in sorted_by_indices]) + lower = 0.0 + X1, Y1 = encoder.transform() assert Y1.tolist() == [[1.0], [5.0]] - assert ((X1 <= 1.0) & (X1 >= 0.0)).all() + assert ((X1 <= upper) & (X1 >= lower)).all() # Log encoder - encoder = RunHistoryLogEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryLogEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryLogScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryLogScaledEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryScaledEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryInverseScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryInverseScaledEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistorySqrtScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistorySqrtScaledEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryEIPSEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryEIPSEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() -def test_transform_conditionals(runhistory, make_scenario, configspace_large, configs): - configs = configspace_large.sample_configuration(20) +def test_transform_conditionals(runhistory, make_scenario, configspace_large): scenario = make_scenario(configspace_large) + config_1 = Configuration( + configspace_large, + values={ + "activation": "tanh", + "n_layer": 5, + "n_neurons": 27, + "solver": "lbfgs", + }, + ) + config_2 = Configuration( + configspace_large, + values={ + "activation": "tanh", + "batch_size": 47, + "learning_rate": "adaptive", + "learning_rate_init": 0.6673206111956781, + "n_layer": 3, + "n_neurons": 88, + "solver": "sgd", + }, + ) runhistory.add( - config=configs[0], + config=config_1, cost=1, time=1, status=StatusType.SUCCESS, ) runhistory.add( - config=configs[2], + config=config_2, cost=5, time=4, status=StatusType.SUCCESS, ) - encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform() @@ -124,7 +184,9 @@ def test_multi_objective(runhistory, make_scenario, configspace_small, configs): # Multi objective algorithm must be set with pytest.raises(AssertionError): - encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory _, Y = encoder.transform() @@ -180,7 +242,9 @@ def test_ignore(runhistory, make_scenario, configspace_small, configs): ) # Normal encoder - encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X1, Y1 = encoder.transform() @@ -219,10 +283,14 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs): budget=2, ) - runhistory.add(config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2) + runhistory.add( + config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2 + ) # Normal encoder - encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform(budget_subset=[2]) assert Y.tolist() == [[99999999]] @@ -251,10 +319,14 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs): budget=2, ) - runhistory.add(config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2) + runhistory.add( + config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2 + ) # Normal encoder - encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform(budget_subset=[2]) assert Y.tolist() == [[99999999]] @@ -266,12 +338,20 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs): def test_lower_budget_states(runhistory, make_scenario, configspace_small, configs): """Tests lower budgets based on budget subset and considered states.""" scenario = make_scenario(configspace_small) - encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory - runhistory.add(config=configs[0], cost=1, time=1, status=StatusType.SUCCESS, budget=3) - runhistory.add(config=configs[0], cost=2, time=2, status=StatusType.SUCCESS, budget=4) - runhistory.add(config=configs[0], cost=3, time=4, status=StatusType.TIMEOUT, budget=5) + runhistory.add( + config=configs[0], cost=1, time=1, status=StatusType.SUCCESS, budget=3 + ) + runhistory.add( + config=configs[0], cost=2, time=2, status=StatusType.SUCCESS, budget=4 + ) + runhistory.add( + config=configs[0], cost=3, time=4, status=StatusType.TIMEOUT, budget=5 + ) # We request a higher budget but can't find it, so we expect an empty list X, Y = encoder.transform(budget_subset=[500]) From 5fd67a456415eab7839f1dbb8c0767993ddf14ba Mon Sep 17 00:00:00 2001 From: helegraf Date: Thu, 16 May 2024 14:19:06 +0200 Subject: [PATCH 30/57] Change stale-reminder to recent-reminder (#1095) (#1107) From 829cddeb4805b5ec3f71e249b9d6ab8ce8f98fbf Mon Sep 17 00:00:00 2001 From: fleance Date: Thu, 16 May 2024 14:32:41 +0200 Subject: [PATCH 31/57] Fix typo in docstring for `MultiFideltyFacade.get_initial_design` (#1104) * Fix typo in docstring for `MultiFideltyFacade.get_initial_design` The docstring specifies 0.1 is used as a default value for max_ratio, however in code 0.25 is used. --- smac/facade/multi_fidelity_facade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smac/facade/multi_fidelity_facade.py b/smac/facade/multi_fidelity_facade.py index 1e46b4170..717162c09 100644 --- a/smac/facade/multi_fidelity_facade.py +++ b/smac/facade/multi_fidelity_facade.py @@ -80,7 +80,7 @@ def get_initial_design( # type: ignore Number of initial configurations per hyperparameter. For example, if my configuration space covers five hyperparameters and ``n_configs_per_hyperparameter`` is set to 10, then 50 initial configurations will be samples. - max_ratio: float, defaults to 0.1 + max_ratio: float, defaults to 0.25 Use at most ``scenario.n_trials`` * ``max_ratio`` number of configurations in the initial design. Additional configurations are not affected by this parameter. additional_configs: list[Configuration], defaults to [] From a5ab1190000f8113891b54e3afd2bc3ea6c311f6 Mon Sep 17 00:00:00 2001 From: "C. Benjamins" <75323339+benjamc@users.noreply.github.com> Date: Thu, 16 May 2024 14:34:03 +0200 Subject: [PATCH 32/57] Bug/acquisition maximizer sampling (#1106) * Adapt how n_points works in the config_selector.py Also remove LocalAndSortedPriorRandomSearch by integrating it into LocalAndSortedRandomSearch * Change retrain_after to 1 for BlackBOx Facade * Fix bugs in local_and_random_search.py * Edit CHANGELOG.md * fix acq maximizer docstring * call get config selector from super class * Update random and local search to new functionality * Update CHANGELOG.md * Fix sampling of initial points for local search when less previous configs than n_points * Re-format blackbox_facade.py --------- Co-authored-by: Helena Graf Co-authored-by: dengdifan --- CHANGELOG.md | 13 +- smac/acquisition/maximizer/__init__.py | 2 - .../abstract_acqusition_maximizer.py | 13 +- .../maximizer/differential_evolution.py | 1 + smac/acquisition/maximizer/helpers.py | 47 +--- .../maximizer/local_and_random_search.py | 231 +++++++----------- smac/acquisition/maximizer/local_search.py | 29 +-- smac/facade/blackbox_facade.py | 13 + smac/main/config_selector.py | 1 - tests/test_acquisition/test_maximizers.py | 28 +-- 10 files changed, 147 insertions(+), 231 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d3e5e158..f6768b689 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,11 @@ -# 2.0.3 +# 2.1.0 + +## Improvements +- Change the surrogate model to be retrained after every iteration by default in the case of blackbox optimization + (#1106). +- Integrate `LocalAndSortedPriorRandomSearch` functionality into `LocalAndSortedRandomSearch` (#1106). +- Change the way the `LocalAndSortedRandomSearch` works such that the incumbent always is a starting point and that + random configurations are sampled as the basis of the local search, not in addition (#1106). ## Bugfixes - Fix path for dask scheduler file (#1055). @@ -7,6 +14,10 @@ - Propagate the Scenario random seed to `get_random_design` (#1066). - Configurations that fail to become incumbents will be added to the rejected lists (#1069). - SMAC RandomForest doesn't crash when `np.integer` used, i.e. as generated from a `np.random.RandomState` (#1084). +- Fix the handling of n_points/ challengers in the acquisition maximizers, such that this number now functions as the + number of points that are sampled from the acquisition function to find the next challengers. Now also doesn't + restrict the config selector to n_retrain many points for finding the max, and instead uses the defaults that are + defined via facades/ scenarios (#1106). ## Misc - ci: Update action version (#1072). diff --git a/smac/acquisition/maximizer/__init__.py b/smac/acquisition/maximizer/__init__.py index 6fea59b45..5e3756190 100644 --- a/smac/acquisition/maximizer/__init__.py +++ b/smac/acquisition/maximizer/__init__.py @@ -3,7 +3,6 @@ ) from smac.acquisition.maximizer.differential_evolution import DifferentialEvolution from smac.acquisition.maximizer.local_and_random_search import ( - LocalAndSortedPriorRandomSearch, LocalAndSortedRandomSearch, ) from smac.acquisition.maximizer.local_search import LocalSearch @@ -13,7 +12,6 @@ "AbstractAcquisitionMaximizer", "DifferentialEvolution", "LocalAndSortedRandomSearch", - "LocalAndSortedPriorRandomSearch", "LocalSearch", "RandomSearch", ] diff --git a/smac/acquisition/maximizer/abstract_acqusition_maximizer.py b/smac/acquisition/maximizer/abstract_acqusition_maximizer.py index 5a14a8814..e148cb6ed 100644 --- a/smac/acquisition/maximizer/abstract_acqusition_maximizer.py +++ b/smac/acquisition/maximizer/abstract_acqusition_maximizer.py @@ -27,12 +27,9 @@ class AbstractAcquisitionMaximizer: Parameters ---------- - configspace : ConfigurationSpace - acquisition_function : AbstractAcquisitionFunction - challengers : int, defaults to 5000 - Number of configurations to sample from the configuration space to get - the acquisition function value for, thus challenging the current - incumbent and becoming a candidate for the next function evaluation. + configspace : ConfigurationSpace acquisition_function : AbstractAcquisitionFunction + challengers : int, defaults to 5000 Number of configurations sampled during the optimization process, + details depend on the used maximizer. Also, the number of configurations that is returned by calling `maximize`. seed : int, defaults to 0 """ @@ -85,8 +82,8 @@ def maximize( previous_configs: list[Configuration] Previous evaluated configurations. n_points: int, defaults to None - Number of points to be sampled. If `n_points` is not specified, - `self._challengers` is used. + Number of points to be sampled & number of configurations to be returned. If `n_points` is not specified, + `self._challengers` is used. Semantics depend on concrete implementation. random_design: AbstractRandomDesign, defaults to None Part of the returned ChallengerList such that we can interleave random configurations by a scheme defined by the random design. The method `random_design.next_iteration()` diff --git a/smac/acquisition/maximizer/differential_evolution.py b/smac/acquisition/maximizer/differential_evolution.py index d1201c19c..0f2ce15e6 100644 --- a/smac/acquisition/maximizer/differential_evolution.py +++ b/smac/acquisition/maximizer/differential_evolution.py @@ -30,6 +30,7 @@ def _maximize( previous_configs: list[Configuration], n_points: int, ) -> list[tuple[float, Configuration]]: + # n_points is not used here, but is required by the interface configs: list[tuple[float, Configuration]] = [] diff --git a/smac/acquisition/maximizer/helpers.py b/smac/acquisition/maximizer/helpers.py index d7319f20f..db06e2aa4 100644 --- a/smac/acquisition/maximizer/helpers.py +++ b/smac/acquisition/maximizer/helpers.py @@ -4,8 +4,8 @@ from ConfigSpace import Configuration, ConfigurationSpace +from smac.random_design import ProbabilityRandomDesign from smac.random_design.abstract_random_design import AbstractRandomDesign -from smac.random_design.modulus_design import ModulusRandomDesign class ChallengerList(Iterator): @@ -20,7 +20,7 @@ class ChallengerList(Iterator): ---------- configspace : ConfigurationSpace challenger_callback : Callable - Callback function which returns a list of challengers (without interleaved random configurations, must a be a + Callback function which returns a list of challengers (without interleaved random configurations), must a be a python closure. random_design : AbstractRandomDesign | None, defaults to ModulusRandomDesign(modulus=2.0) Which random design should be used. @@ -30,7 +30,7 @@ def __init__( self, configspace: ConfigurationSpace, challenger_callback: Callable, - random_design: AbstractRandomDesign | None = ModulusRandomDesign(modulus=2.0), + random_design: AbstractRandomDesign | None = ProbabilityRandomDesign(seed=0, probability=0.08447232371720552), ): self._challengers_callback = challenger_callback self._challengers: list[Configuration] | None = None @@ -72,44 +72,3 @@ def __len__(self) -> int: self._challengers = self._challengers_callback() return len(self._challengers) - self._index - - -''' -class FixedSet(AbstractAcquisitionMaximizer): - def __init__( - self, - configurations: list[Configuration], - acquisition_function: AbstractAcquisitionFunction, - configspace: ConfigurationSpace, - challengers: int = 5000, - seed: int = 0, - ): - """Maximize the acquisition function over a finite list of configurations. - - Parameters - ---------- - configurations : list[~smac._configspace.Configuration] - Candidate configurations - acquisition_function : ~smac.acquisition.AbstractAcquisitionFunction - - configspace : ~smac._configspace.ConfigurationSpace - - rng : np.random.RandomState or int, optional - """ - super().__init__( - acquisition_function=acquisition_function, configspace=configspace, challengers=challengers, seed=seed - ) - self.configurations = configurations - - def _maximize( - self, - runhistory: RunHistory, - stats: Stats, - n_points: int, - ) -> list[tuple[float, Configuration]]: - configurations = copy.deepcopy(self.configurations) - for config in configurations: - config.origin = "Fixed Set" - - return self._sort_by_acquisition_value(configurations) -''' diff --git a/smac/acquisition/maximizer/local_and_random_search.py b/smac/acquisition/maximizer/local_and_random_search.py index bb6daaef5..d81af2f69 100644 --- a/smac/acquisition/maximizer/local_and_random_search.py +++ b/smac/acquisition/maximizer/local_and_random_search.py @@ -21,7 +21,7 @@ class LocalAndSortedRandomSearch(AbstractAcquisitionMaximizer): """Implement SMAC's default acquisition function optimization. - This optimizer performs local search from the previous best points according, to the acquisition + This optimizer performs local search from the previous best points according to the acquisition function, uses the acquisition function to sort randomly sampled configurations. Random configurations are interleaved by the main SMAC code. @@ -31,6 +31,10 @@ class LocalAndSortedRandomSearch(AbstractAcquisitionMaximizer): Parameters ---------- configspace : ConfigurationSpace + uniform_configspace : ConfigurationSpace + A version of the user-defined ConfigurationSpace where all parameters are uniform (or have their weights removed + in the case of a categorical hyperparameter). Can optionally be given and sampling ratios be defined via the + `prior_sampling_fraction` parameter. acquisition_function : AbstractAcquisitionFunction | None, defaults to None challengers : int, defaults to 5000 Number of challengers. @@ -40,6 +44,9 @@ class LocalAndSortedRandomSearch(AbstractAcquisitionMaximizer): [LocalSearch] number of steps during a plateau walk before local search terminates. local_search_iterations: int, defauts to 10 [Local Search] number of local search iterations. + prior_sampling_fraction: float, defaults to 0.5 + The ratio of random samples that are taken from the user-defined ConfigurationSpace, as opposed to the uniform + version (needs `uniform_configspace`to be defined). seed : int, defaults to 0 """ @@ -52,6 +59,8 @@ def __init__( n_steps_plateau_walk: int = 10, local_search_iterations: int = 10, seed: int = 0, + uniform_configspace: ConfigurationSpace | None = None, + prior_sampling_fraction: float | None = None, ) -> None: super().__init__( configspace, @@ -60,11 +69,28 @@ def __init__( seed=seed, ) - self._random_search = RandomSearch( - configspace=configspace, - acquisition_function=acquisition_function, - seed=seed, - ) + if uniform_configspace is not None and prior_sampling_fraction is None: + prior_sampling_fraction = 0.5 + if uniform_configspace is None and prior_sampling_fraction is not None: + raise ValueError("If `prior_sampling_fraction` is given, `uniform_configspace` must be defined.") + if uniform_configspace is not None and prior_sampling_fraction is not None: + self._prior_random_search = RandomSearch( + acquisition_function=acquisition_function, + configspace=configspace, + seed=seed, + ) + + self._uniform_random_search = RandomSearch( + acquisition_function=acquisition_function, + configspace=uniform_configspace, + seed=seed, + ) + else: + self._random_search = RandomSearch( + configspace=configspace, + acquisition_function=acquisition_function, + seed=seed, + ) self._local_search = LocalSearch( configspace=configspace, @@ -75,6 +101,8 @@ def __init__( ) self._local_search_iterations = local_search_iterations + self._prior_sampling_fraction = prior_sampling_fraction + self._uniform_configspace = uniform_configspace @property def acquisition_function(self) -> AbstractAcquisitionFunction | None: # noqa: D102 @@ -84,18 +112,31 @@ def acquisition_function(self) -> AbstractAcquisitionFunction | None: # noqa: D @acquisition_function.setter def acquisition_function(self, acquisition_function: AbstractAcquisitionFunction) -> None: self._acquisition_function = acquisition_function - self._random_search._acquisition_function = acquisition_function + if self._uniform_configspace is not None: + self._prior_random_search._acquisition_function = acquisition_function + self._uniform_random_search._acquisition_function = acquisition_function + else: + self._random_search._acquisition_function = acquisition_function self._local_search._acquisition_function = acquisition_function @property def meta(self) -> dict[str, Any]: # noqa: D102 meta = super().meta - meta.update( - { - "random_search": self._random_search.meta, - "local_search": self._local_search.meta, - } - ) + if self._uniform_configspace is None: + meta.update( + { + "random_search": self._random_search.meta, + "local_search": self._local_search.meta, + } + ) + else: + meta.update( + { + "prior_random_search": self._prior_random_search.meta, + "uniform_random_search": self._uniform_random_search.meta, + "local_search": self._local_search.meta, + } + ) return meta @@ -105,147 +146,45 @@ def _maximize( n_points: int, ) -> list[tuple[float, Configuration]]: - # Get configurations sorted by EI - next_configs_by_random_search_sorted = self._random_search._maximize( - previous_configs=previous_configs, - n_points=n_points, - _sorted=True, - ) - + if self._uniform_configspace is not None and self._prior_sampling_fraction is not None: + # Get configurations sorted by acquisition function value + next_configs_by_prior_random_search_sorted = self._prior_random_search._maximize( + previous_configs, + round(n_points * self._prior_sampling_fraction), + _sorted=True, + ) + + # Get configurations sorted by acquisition function value + next_configs_by_uniform_random_search_sorted = self._uniform_random_search._maximize( + previous_configs, + round(n_points * (1 - self._prior_sampling_fraction)), + _sorted=True, + ) + next_configs_by_random_search_sorted = ( + next_configs_by_uniform_random_search_sorted + next_configs_by_prior_random_search_sorted + ) + next_configs_by_random_search_sorted.sort(reverse=True, key=lambda x: x[0]) + else: + # Get configurations sorted by acquisition function value + next_configs_by_random_search_sorted = self._random_search._maximize( + previous_configs=previous_configs, + n_points=n_points, + _sorted=True, + ) + + # Choose the best self._local_search_iterations random configs to start the local search, and choose only + # incumbent from previous configs + random_starting_points = next_configs_by_random_search_sorted[: self._local_search_iterations] next_configs_by_local_search = self._local_search._maximize( previous_configs=previous_configs, n_points=self._local_search_iterations, - additional_start_points=next_configs_by_random_search_sorted, + additional_start_points=random_starting_points, ) - # Having the configurations from random search, sorted by their - # acquisition function value is important for the first few iterations - # of SMAC. As long as the random forest predicts constant value, we - # want to use only random configurations. Having them at the begging of - # the list ensures this (even after adding the configurations by local - # search, and then sorting them) - next_configs_by_acq_value = next_configs_by_random_search_sorted + next_configs_by_local_search + next_configs_by_acq_value = next_configs_by_local_search next_configs_by_acq_value.sort(reverse=True, key=lambda x: x[0]) first_five = [f"{_[0]} ({_[1].origin})" for _ in next_configs_by_acq_value[:5]] logger.debug(f"First 5 acquisition function values of selected configurations:\n{', '.join(first_five)}") return next_configs_by_acq_value - - -class LocalAndSortedPriorRandomSearch(AbstractAcquisitionMaximizer): - """Implements SMAC's default acquisition function optimization. - - This optimizer performs local search from the previous best points according to the acquisition function, uses the - acquisition function to sort randomly sampled configurations. Random configurations are interleaved by the main SMAC - code. The random configurations are retrieved from two different ConfigurationSpaces - one which uses priors - (e.g. NormalFloatHP) and is defined by the user, and one that is a uniform version of the same space, i.e. with the - priors removed. - - Parameters - ---------- - configspace : ConfigurationSpace - The original ConfigurationSpace specified by the user. - uniform_configspace : ConfigurationSpace - A version of the user-defined ConfigurationSpace where all parameters are uniform (or have their weights removed - in the case of a categorical hyperparameter). - acquisition_function : AbstractAcquisitionFunction | None, defaults to None - challengers : int, defaults to 5000 - Number of challengers. - max_steps: int, defaults to None - [LocalSearch] Maximum number of steps that the local search will perform. - n_steps_plateau_walk: int, defaults to 10 - [LocalSearch] number of steps during a plateau walk before local search terminates. - local_search_iterations: int, defaults to 10 - [Local Search] number of local search iterations. - prior_sampling_fraction: float, defaults to 0.5 - The ratio of random samples that are taken from the user-defined ConfigurationSpace, as opposed to the uniform - version. - seed : int, defaults to 0 - """ - - def __init__( - self, - configspace: ConfigurationSpace, - uniform_configspace: ConfigurationSpace, - acquisition_function: AbstractAcquisitionFunction | None = None, - challengers: int = 5000, - max_steps: int | None = None, - n_steps_plateau_walk: int = 10, - local_search_iterations: int = 10, - prior_sampling_fraction: float = 0.5, - seed: int = 0, - ) -> None: - super().__init__( - acquisition_function, - configspace, - challengers=challengers, - seed=seed, - ) - - self._prior_random_search = RandomSearch( - acquisition_function=acquisition_function, - configspace=configspace, - seed=seed, - ) - - self._uniform_random_search = RandomSearch( - acquisition_function=acquisition_function, - configspace=uniform_configspace, - seed=seed, - ) - - self._local_search = LocalSearch( - acquisition_function=acquisition_function, - configspace=configspace, - max_steps=max_steps, - n_steps_plateau_walk=n_steps_plateau_walk, - seed=seed, - ) - - self._local_search_iterations = local_search_iterations - self._prior_sampling_fraction = prior_sampling_fraction - - def _maximize( - self, - previous_configs: list[Configuration], - n_points: int, - ) -> list[tuple[float, Configuration]]: - - # Get configurations sorted by EI - next_configs_by_prior_random_search_sorted = self._prior_random_search._maximize( - previous_configs, - round(n_points * self._prior_sampling_fraction), - _sorted=True, - ) - - # Get configurations sorted by EI - next_configs_by_uniform_random_search_sorted = self._uniform_random_search._maximize( - previous_configs, - round(n_points * (1 - self._prior_sampling_fraction)), - _sorted=True, - ) - next_configs_by_random_search_sorted = [] - next_configs_by_random_search_sorted.extend(next_configs_by_prior_random_search_sorted) - next_configs_by_random_search_sorted.extend(next_configs_by_uniform_random_search_sorted) - - next_configs_by_local_search = self._local_search._maximize( - previous_configs, - self._local_search_iterations, - additional_start_points=next_configs_by_random_search_sorted, - ) - - # Having the configurations from random search, sorted by their - # acquisition function value is important for the first few iterations - # of SMAC. As long as the random forest predicts constant value, we - # want to use only random configurations. Having them at the begging of - # the list ensures this (even after adding the configurations by local - # search, and then sorting them) - next_configs_by_acq_value = next_configs_by_random_search_sorted + next_configs_by_local_search - next_configs_by_acq_value.sort(reverse=True, key=lambda x: x[0]) - logger.debug( - "First 5 acq func (origin) values of selected configurations: %s", - str([[_[0], _[1].origin] for _ in next_configs_by_acq_value[:5]]), - ) - - return next_configs_by_acq_value diff --git a/smac/acquisition/maximizer/local_search.py b/smac/acquisition/maximizer/local_search.py index c6f545f9a..3ef1ae96e 100644 --- a/smac/acquisition/maximizer/local_search.py +++ b/smac/acquisition/maximizer/local_search.py @@ -90,9 +90,9 @@ def _maximize( n_points: int, additional_start_points: list[tuple[float, Configuration]] | None = None, ) -> list[tuple[float, Configuration]]: - """Start a local search from the given startpoints. Iteratively collect neighbours + """Start a local search from the given start points. Iteratively collect neighbours using Configspace.utils.get_one_exchange_neighbourhood and evaluate them. - If the new config is better than the current best, the local search is coninued from the + If the new config is better than the current best, the local search is continued from the new config. Quit if either the max number of steps is reached or @@ -149,16 +149,22 @@ def _get_initial_points( list[Configuration] A list of initial points/configurations. """ - if len(previous_configs) == 0: - init_points = self._configspace.sample_configuration(size=n_points) - else: + sampled_points = [] + init_points = [] + n_init_points = n_points + if len(previous_configs) < n_points: + sampled_points = self._configspace.sample_configuration(size=n_points - len(previous_configs)) + n_init_points = len(previous_configs) + if not isinstance(sampled_points, list): + sampled_points = [sampled_points] + if len(previous_configs) > 0: init_points = self._get_init_points_from_previous_configs( previous_configs, - n_points, + n_init_points, additional_start_points, ) - return init_points + return sampled_points + init_points def _get_init_points_from_previous_configs( self, @@ -187,7 +193,7 @@ def _get_init_points_from_previous_configs( previous_configs: list[Configuration] Previous configuration (e.g., from the runhistory). n_points: int - Number of initial points to be generated. + Number of initial points to be generated; selected from previous configs (+ random configs if necessary). additional_start_points: list[tuple[float, Configuration]] | None Additional starting points. @@ -198,10 +204,6 @@ def _get_init_points_from_previous_configs( """ assert self._acquisition_function is not None - # configurations with the highest previous EI - configs_previous_runs_sorted = self._sort_by_acquisition_value(previous_configs) - configs_previous_runs_sorted = [conf[1] for conf in configs_previous_runs_sorted[:n_points]] - # configurations with the lowest predictive cost, check for None to make unit tests work if self._acquisition_function.model is not None: conf_array = convert_configurations_to_array(previous_configs) @@ -228,14 +230,13 @@ def _get_init_points_from_previous_configs( previous_configs_sorted_by_cost = [] if additional_start_points is not None: - additional_start_points = [asp[1] for asp in additional_start_points[:n_points]] + additional_start_points = [asp[1] for asp in additional_start_points] else: additional_start_points = [] init_points = [] init_points_as_set: set[Configuration] = set() for cand in itertools.chain( - configs_previous_runs_sorted, previous_configs_sorted_by_cost, additional_start_points, ): diff --git a/smac/facade/blackbox_facade.py b/smac/facade/blackbox_facade.py index 33837d127..a5e1a18ab 100644 --- a/smac/facade/blackbox_facade.py +++ b/smac/facade/blackbox_facade.py @@ -11,6 +11,7 @@ from smac.facade.abstract_facade import AbstractFacade from smac.initial_design.sobol_design import SobolInitialDesign from smac.intensifier.intensifier import Intensifier +from smac.main.config_selector import ConfigSelector from smac.model.gaussian_process.abstract_gaussian_process import ( AbstractGaussianProcess, ) @@ -311,3 +312,15 @@ def get_runhistory_encoder( ) -> RunHistoryEncoder: """Returns the default runhistory encoder.""" return RunHistoryEncoder(scenario) + + @staticmethod + def get_config_selector( + scenario: Scenario, + *, + retrain_after: int = 1, + retries: int = 16, + ) -> ConfigSelector: + """Returns the default configuration selector.""" + return super(BlackBoxFacade, BlackBoxFacade).get_config_selector( + scenario, retrain_after=retrain_after, retries=retries + ) diff --git a/smac/main/config_selector.py b/smac/main/config_selector.py index ea57bf0bf..4e3574d58 100644 --- a/smac/main/config_selector.py +++ b/smac/main/config_selector.py @@ -213,7 +213,6 @@ def __iter__(self) -> Iterator[Configuration]: # Now we maximize the acquisition function challengers = self._acquisition_maximizer.maximize( previous_configs, - n_points=self._retrain_after, random_design=self._random_design, ) diff --git a/tests/test_acquisition/test_maximizers.py b/tests/test_acquisition/test_maximizers.py index 4cd4110f3..6d373c237 100644 --- a/tests/test_acquisition/test_maximizers.py +++ b/tests/test_acquisition/test_maximizers.py @@ -30,7 +30,6 @@ from smac.acquisition.function import EI from smac.acquisition.maximizer import ( DifferentialEvolution, - LocalAndSortedPriorRandomSearch, LocalAndSortedRandomSearch, LocalSearch, RandomSearch, @@ -278,7 +277,7 @@ def __call__(self, X): random_configs = configspace.sample_configuration(size=100) points = ls._get_initial_points(random_configs, n_points=5, additional_start_points=None) - assert len(points) == 10 + assert len(points) == 5 # -------------------------------------------------------------- @@ -342,7 +341,6 @@ def test_local_and_random_search(configspace, acquisition_function): assert v_old >= v v_old = v - assert "Acquisition Function Maximizer: Random Search (sorted)" in config_origins assert "Acquisition Function Maximizer: Local Search" in config_origins @@ -390,26 +388,26 @@ def __call__(self, arrays): budget_kwargs = {"max_steps": 2, "n_steps_plateau_walk": 2, "local_search_iterations": 2} - prs_0 = LocalAndSortedPriorRandomSearch( - configspace_prior, - configspace_rosenbrock, - AcquisitionFunction(), + prs_0 = LocalAndSortedRandomSearch( + configspace=configspace_prior, + uniform_configspace=configspace_rosenbrock, + acquisition_function=AcquisitionFunction(), prior_sampling_fraction=0, **budget_kwargs, ) - prs_05 = LocalAndSortedPriorRandomSearch( - configspace_prior, - configspace_rosenbrock, - AcquisitionFunction(), + prs_05 = LocalAndSortedRandomSearch( + configspace=configspace_prior, + uniform_configspace=configspace_rosenbrock, + acquisition_function=AcquisitionFunction(), prior_sampling_fraction=0.9, **budget_kwargs, ) - prs_1 = LocalAndSortedPriorRandomSearch( - configspace_prior, - configspace_rosenbrock, - AcquisitionFunction(), + prs_1 = LocalAndSortedRandomSearch( + configspace=configspace_prior, + uniform_configspace=configspace_rosenbrock, + acquisition_function=AcquisitionFunction(), prior_sampling_fraction=1, **budget_kwargs, ) From f754bf6ed8267bc6158092c2647b7907ce4d91a8 Mon Sep 17 00:00:00 2001 From: Helena Graf Date: Thu, 16 May 2024 14:45:40 +0200 Subject: [PATCH 33/57] Update version number to 2.1.0 --- CITATION.cff | 2 +- Makefile | 2 +- benchmark/src/wrappers/v20.py | 2 +- smac/__init__.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index a984b24c5..ece8325c6 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,7 +9,7 @@ date-released: "2016-08-17" url: "https://automl.github.io/SMAC3/master/index.html" repository-code: "https://github.com/automl/SMAC3" -version: "2.0.2" +version: "2.1.0" type: "software" keywords: diff --git a/Makefile b/Makefile index ba7a4268b..195773082 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ SHELL := /bin/bash NAME := SMAC3 PACKAGE_NAME := smac -VERSION := 2.0.2 +VERSION := 2.1.0 DIR := "${CURDIR}" SOURCE_DIR := ${PACKAGE_NAME} diff --git a/benchmark/src/wrappers/v20.py b/benchmark/src/wrappers/v20.py index 9f2be56dc..3ee376a37 100644 --- a/benchmark/src/wrappers/v20.py +++ b/benchmark/src/wrappers/v20.py @@ -6,7 +6,7 @@ class Version20(Wrapper): - supported_versions: list[str] = ["2.0.2"] + supported_versions: list[str] = ["2.1.0"] def __init__(self, task: Task, seed: int) -> None: super().__init__(task, seed) diff --git a/smac/__init__.py b/smac/__init__.py index ab2781297..3241da062 100644 --- a/smac/__init__.py +++ b/smac/__init__.py @@ -19,7 +19,7 @@ Copyright {datetime.date.today().strftime('%Y')}, Marius Lindauer, Katharina Eggensperger, Matthias Feurer, André Biedenkapp, Difan Deng, Carolin Benjamins, Tim Ruhkopf, René Sass and Frank Hutter""" -version = "2.0.2" +version = "2.1.0" try: From 937eb2cae720656d741d5ab26a68d3ade348d046 Mon Sep 17 00:00:00 2001 From: "C. Benjamins" <75323339+benjamc@users.noreply.github.com> Date: Thu, 16 May 2024 15:11:43 +0200 Subject: [PATCH 34/57] Version 2.1.0 (#1108) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Hacking in the sum cost for MO * Fix mypi * Bugfix * Make mypi happy * Fix tests * Make format * Remove num_obj in MO optimizer * A lot of restructuring * Solve importings * Solve importings * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Updated changelog * Still fixing :) * fix docstrings * Change directory of tests * Fix ambiguous variable * Del newlines * fix mypy * mypy and doc fix * Removed int type checking * fix tests * Always return List of incs * fix path issue in example * maint boing facade * Better docstring * Return values if no bounds passed * Removed import * Return only one incumbent * Rename n_optimizers -> n_workers * Update example * Update docstring * Remove legacy function * Update to new signature * Update changelog.md * Format black * Update test * Please pre-commit * Merge development * Fix tests * Fix typos * Format black * Updated text * Increased version * Improved readme * SMAC logo 50% * SMAC logo 50% * Removed dev-2.0 * Added doc workflow for latest tag * Fixing docs latest tag * Fixing docs latest tag * First steps new intensifier * Make it work again * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * Test issues * FIX formatting * Link to old documentation * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * FIX formatting * Fix unit tests * Add annotation * corrected log prob for tophat prior * Removed configurations from callback * Fixed tests * Fix mypy * Save docs as latest version too * Enabled all tests again * Make version an env * Added last version to docs * Global variable fix * Added v before version number * New Intensifier Implementation (#888) - Completely reimplemented the intensifiers (including Successive Halving and Hyperband): All intensifiers support multi-fidelity, multi-objective and multi-threading by nature now. - Expected behaviour for ask-and-tell interface ensured (also for Successive Halving). - Continuing a run is now fully supported. - Added more examples. - Updated documentation based on new implementation. * First benchmark steps * Added a warning * Set xi to 0.01 * Incorporated walltime in trajectory * Working version of benchmark * Changed requirements * Changed xi back to 0.0 again * No duplicate budgets * Fixed bug which updated tracker wrongly * Bugfix: Delete correct bracket+stage * Fixed tests * Added costs to the trajectory * Changed initial design max ratio to 0.25 * Make format * Added more details to intensifier * Finished benchmark * Added benchmark source files * Fixed tests because of initial design changes * Improved runhistory API * Fixed typo (#894) * Merge doc fixes * Doc fixes (#893) Co-authored-by: timruhkopf Co-authored-by: Carolin Benjamins Co-authored-by: dengdifan Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs * Make format * Satisfied mypy * Enabled more tests * Increased version * Mypy * Mypy * Mypy * Fixed critical bug when using highest budget for inc selection * Added SH+HB comparison * Stuff * Updated report * Fixed docstring * Fixed image path * Removed vscode folder * Bugfixes * Doc fixes * Updated changelog * Make isort * Fix typos * Removed new line * Added alpha2 to docs selection * update description * Fix 531 correctly create integers in initial design This PR fixes a bug in which integer hyperparameters would not be created correctly in the initial design. This went unnoticed, except in the case where it was part of a condition and the function deactivate_inactive_hyperparameters was called. This PR fixes this bug by mapping the random float that was sampled by, for example and LHD or Sobol sequence, to the float representation of an integer by calling _transform and _inverse_transform of the integer hyperparameter. * Add contact paragraph to README. * Increase days until stale issues are closed by stalebot (#904) * Edit changelog * Add issue number * Closes #907 * Clarify origin of configurations (#908) * Random forest speed up (#903) Co-authored-by: René Sass * Add: workflow to update pre-commit versions (#874) Co-authored-by: René Sass * Add Git-Flow Workflow Specification * [DOC] acquisition function maximizers * Add stale reminder email action * Increase version number of ConfigSpace * Add guideline for updating copyright notice years to CONTRIBUTING.md * Update requirement for pyrfr (#920) * Update pre-commit: isort,flake8 version numbers * Format black * Fix mypy * Ac bench (#929) * Add Branin AC Task * Add ac optimization type * Nicer plotting * Add ac mode * Update ac task * Make new smac env name adjustable * Increase python version for smac 1.4 * Update benchmark * Update plotting * Goodby comments * Fix path * Fix 1.4 wrapper * Update benchmark results --------- Co-authored-by: Carolin Benjamins * [DOC] added documentation on the incumbent update * Feature/add version to make publish (#931) * Add version, correct package name and use bash in makefile * Increase version * Increase version in makefile, add places with versions to make publish * Fix typo * Transform integer feature names to strings (#930) * Transform int features to str * Fix pre-commit * Update version to 2.0.0 * Update version to 2.0.0 * Update CHANGELOG.md * Add information on deepcave to the readme * Correct doc version * fix(runhistory): set id of the sampled config (#951) * fix: Set config_id of config in runhistory * ci: Set dependabot to target development branch (#963) * feat: Option preventing SMAC setting up logging (#947) * Version 2.0.0 (#898) * Update runhistory2epm.py * pre_commit * Updated changelog * Fixed comment * Updated changelog * Some small improvements * Removed trailing space * Updated changelog * Restricted pynisher * Updated changelog * Link JMLR paper * Update bibtex entry Before: arxiv paper, now journal article. * Fixed citation in docs * Fixed callback typos * BOinG for mixed hyperparameter search space * fix format * maint * maint documentation * maint pre-commit * fix workflow * maint * Add dependabot for workflow files * Bump actions/setup-python from 2 to 4 (#849) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/checkout from 2 to 3 (#850) Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * maint * Updated stale bot * Updated stale options again * Fixed typing * Increased version * rename check_point_in_ss to check_subspace_point * Make format * Uncomment test procedure * docstrings for boing facade * avoid typing namespace everywhere in epm_configuration_chooser * fix docstring in rh2epm_boing * Limit on dask * Trying to fix tests * Replaced typing in all files * docstring for boing subspace * Changed feature requests to feature * allow acq_optimizer_local_kwargs to be passed to boing subspace without acq_optimizer_local * fix pre-commit * Added any * Update facade * Fix typing * Match arguments from pSMAC with SMAC4AC signature * Add description of pSMAC to docstring * Add option to pass facade class to pSMAC * Add example for pSMAC * Removed docstring in readme's example * Corrected text * move construct_gp_kernels to smac.epm_gpytorch.boing_kernels * maint * fix wrongly imported func name in tests * Add reference for pSMAC #609 * Rename example file * Added roadmap to readme * Prettification * Add result merger * Reformat with black * Delete unnecessary import * Fix typing * Inject first trajectory entry from file from first rundir * Add test for ResultMerger * Fix docstring * Get runhistory and trajectory from pSMAC run Via the result merger. Searchers rundirs and merges found runhistories and calculates a new trajectory. * Get trajectory * Add regex to requirements * Fixed MO cache problem * Fix MO tests * Fixing MO in AbstractRacer * Accepting ints * Hacking in the sum cost for MO * Fix mypi * Bugfix * Make mypi happy * Fix tests * Make format * Remove num_obj in MO optimizer * A lot of restructuring * Solve importings * Solve importings * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Updated changelog * Still fixing :) * fix docstrings * Change directory of tests * Fix ambiguous variable * Del newlines * fix mypy * mypy and doc fix * Removed int type checking * fix tests * Always return List of incs * fix path issue in example * maint boing facade * Better docstring * Return values if no bounds passed * Removed import * Return only one incumbent * Rename n_optimizers -> n_workers * Update example * Update docstring * Remove legacy function * Update to new signature * Update changelog.md * Format black * Update test * Please pre-commit * Merge development * Fix tests * Fix typos * Format black * Updated text * Increased version * Improved readme * SMAC logo 50% * SMAC logo 50% * Removed dev-2.0 * Added doc workflow for latest tag * Fixing docs latest tag * Fixing docs latest tag * First steps new intensifier * Make it work again * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * Test issues * FIX formatting * Link to old documentation * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * FIX formatting * Fix unit tests * Add annotation * corrected log prob for tophat prior * Removed configurations from callback * Fixed tests * Fix mypy * Save docs as latest version too * Enabled all tests again * Make version an env * Added last version to docs * Global variable fix * Added v before version number * New Intensifier Implementation (#888) - Completely reimplemented the intensifiers (including Successive Halving and Hyperband): All intensifiers support multi-fidelity, multi-objective and multi-threading by nature now. - Expected behaviour for ask-and-tell interface ensured (also for Successive Halving). - Continuing a run is now fully supported. - Added more examples. - Updated documentation based on new implementation. * First benchmark steps * Added a warning * Set xi to 0.01 * Incorporated walltime in trajectory * Working version of benchmark * Changed requirements * Changed xi back to 0.0 again * No duplicate budgets * Fixed bug which updated tracker wrongly * Bugfix: Delete correct bracket+stage * Fixed tests * Added costs to the trajectory * Changed initial design max ratio to 0.25 * Make format * Added more details to intensifier * Finished benchmark * Added benchmark source files * Fixed tests because of initial design changes * Improved runhistory API * Fixed typo (#894) * Merge doc fixes * Doc fixes (#893) Co-authored-by: timruhkopf Co-authored-by: Carolin Benjamins Co-authored-by: dengdifan Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs * Make format * Satisfied mypy * Enabled more tests * Increased version * Mypy * Mypy * Mypy * Fixed critical bug when using highest budget for inc selection * Added SH+HB comparison * Stuff * Updated report * Fixed docstring * Fixed image path * Removed vscode folder * Bugfixes * Doc fixes * Updated changelog * Make isort * Fix typos * Removed new line * Added alpha2 to docs selection * update description * Fix 531 correctly create integers in initial design This PR fixes a bug in which integer hyperparameters would not be created correctly in the initial design. This went unnoticed, except in the case where it was part of a condition and the function deactivate_inactive_hyperparameters was called. This PR fixes this bug by mapping the random float that was sampled by, for example and LHD or Sobol sequence, to the float representation of an integer by calling _transform and _inverse_transform of the integer hyperparameter. * Add contact paragraph to README. * Increase days until stale issues are closed by stalebot (#904) * Edit changelog * Add issue number * Closes #907 * Clarify origin of configurations (#908) * Random forest speed up (#903) Co-authored-by: René Sass * Add: workflow to update pre-commit versions (#874) Co-authored-by: René Sass * Add Git-Flow Workflow Specification * [DOC] acquisition function maximizers * Add stale reminder email action * Increase version number of ConfigSpace * Add guideline for updating copyright notice years to CONTRIBUTING.md * Update requirement for pyrfr (#920) * Update pre-commit: isort,flake8 version numbers * Format black * Fix mypy * Ac bench (#929) * Add Branin AC Task * Add ac optimization type * Nicer plotting * Add ac mode * Update ac task * Make new smac env name adjustable * Increase python version for smac 1.4 * Update benchmark * Update plotting * Goodby comments * Fix path * Fix 1.4 wrapper * Update benchmark results --------- Co-authored-by: Carolin Benjamins * [DOC] added documentation on the incumbent update * Feature/add version to make publish (#931) * Add version, correct package name and use bash in makefile * Increase version * Increase version in makefile, add places with versions to make publish * Fix typo * Transform integer feature names to strings (#930) * Transform int features to str * Fix pre-commit * Update version to 2.0.0 * Update version to 2.0.0 * Update CHANGELOG.md * Add information on deepcave to the readme * Correct doc version --------- Co-authored-by: dengdifan <33290713+dengdifan@users.noreply.github.com> Co-authored-by: dengdifan Co-authored-by: Matthias Feurer Co-authored-by: Carolin Benjamins Co-authored-by: eddiebergman Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Tim Ruhkopf * feat: Option preventing SMAC setting up logging * fix: Run isort * Bump peter-evans/create-pull-request from 3 to 5 (#958) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 3 to 5. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v3...v5) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/setup-python from 2 to 4 (#945) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat: Option preventing SMAC setting up logging * fix: Run isort * fix: Add to changelog * feat: Option preventing SMAC setting up logging * fix: Run isort --------- Signed-off-by: dependabot[bot] Co-authored-by: René Sass Co-authored-by: dengdifan <33290713+dengdifan@users.noreply.github.com> Co-authored-by: dengdifan Co-authored-by: Matthias Feurer Co-authored-by: Carolin Benjamins Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Tim Ruhkopf * Citation update (#961) * update CITATION.cff * update order in CITATION.cff * fix citation date * fix citation * maint * Adjust hyperband configuration distribution across brackets * Compute the actual differences between the isb keys. (#957) * Compute the actual differences between the isb keys. * Added change to CHANGELOG.md * Adjust schedule for stale reminder * fix(logging): Prevent automatic logging setup at init (#970) * fix(logging): Prevent automatic logging setup at init * fix(logging): Rely on `setup_logging` to handle arg * Fix validate method of smbo, update docstring. * Allow callbacks to be added to a specific index, make callback registering public * Update CHANGELOG.md * Fix broken references (#988) * Adapt developer install instructions to include pre-commit installation (#994) * [Feature] Initial Design now supports executing the default config provided by the user by means of the configspace he generated. (and is no longer ignored) * Pipe dask_client through facade for exec on hpc (#983) * Pipe dask_client through facade for exec on hpc * Handle pynisher's memory arg Before: Could only pass int via SMAC with the default unit B, but now tuple/list is possible * Update CHANGELOG.md * Make format * No printi --------- Co-authored-by: Carolin Benjamins * [Bug-fix] Pr was failing due to mutable additional configs default * [doc-fix] the count of initial design configs is computed slightly differently * Feature/metadata callback (#999) * Add example for using a callback to log run metadata to a file * Add issue number in changelog * Changelog formatting * Simplify metadata example * Simplify metadata example * Put callback in smac directory to allow importing it in custom repos * Move callback files to callback directory * Edit changelog * Correct metadata callback argument types * Additional metadata arguments in example * Set metadata default values in callback * Edit comment to fix PR * Documentation/dask client example (#1001) * Update requirements (dask_jobqueue) * Fix broken link * Ignore parallel example * Add parallelization example * Remove imports, add comment abt dask client * Add E's information * Update docs * Add comment about n_workers --------- Co-authored-by: Carolin Benjamins * Documentation/dask client example (#1001) * Update requirements (dask_jobqueue) * Fix broken link * Ignore parallel example * Add parallelization example * Remove imports, add comment abt dask client * Add E's information * Update docs * Add comment about n_workers --------- Co-authored-by: Carolin Benjamins * Feature/dask client (#1002) * Add info message that scenario.n_workers is ignored When passing a dask client, directly set n_workers in client. * Change info to warning --------- Co-authored-by: Carolin Benjamins * Update smac version (#1003) * Update smac version * Add hint on github release in make publish * Update copyright year * check if config in rh when storing state (#997) * check if config in rh when storing state * maint * remove unnecessary changes * Adapt developer install instructions to include pre-commit installation (#994) * Pipe dask_client through facade for exec on hpc (#983) * Pipe dask_client through facade for exec on hpc * Handle pynisher's memory arg Before: Could only pass int via SMAC with the default unit B, but now tuple/list is possible * Update CHANGELOG.md * Make format * No printi --------- Co-authored-by: Carolin Benjamins * update changelog * Increase tolerance for runhistory restoration test --------- Co-authored-by: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: Helena Graf * Add a workaround to be able to pass a dataset via dask.scatter (#993) * [feat] Support dask.scatter * [fix] Add shared_data arg in abstract_facade * Address the comments by Eddie * [fix] Fix errors raised by Eddie * [format] Apply black locally * [fix] Get retries back to 16 as it was a mistake * [doc] Update CHANGELOG --------- Co-authored-by: helegraf Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> * Adding Binder links for example (#976) * binder * maint * fix dependencies * maint docs * Revert "maint docs" This reverts commit 4fa1a0fae8dfa0b646e43273d50f72ddd76bf191. * add requirement --------- Co-authored-by: Difan Deng * Remove commit message from dependabot PRs (#1009) * Bugfix/fix broken link (#1012) * Remove commit message from dependabot PRs (#1010) * rename check_point_in_ss to check_subspace_point * Make format * Uncomment test procedure * docstrings for boing facade * avoid typing namespace everywhere in epm_configuration_chooser * fix docstring in rh2epm_boing * Limit on dask * Trying to fix tests * Replaced typing in all files * docstring for boing subspace * Changed feature requests to feature * allow acq_optimizer_local_kwargs to be passed to boing subspace without acq_optimizer_local * fix pre-commit * Added any * Update facade * Fix typing * Match arguments from pSMAC with SMAC4AC signature * Add description of pSMAC to docstring * Add option to pass facade class to pSMAC * Add example for pSMAC * Removed docstring in readme's example * Corrected text * move construct_gp_kernels to smac.epm_gpytorch.boing_kernels * maint * fix wrongly imported func name in tests * Add reference for pSMAC #609 * Rename example file * Added roadmap to readme * Prettification * Add result merger * Reformat with black * Delete unnecessary import * Fix typing * Inject first trajectory entry from file from first rundir * Add test for ResultMerger * Fix docstring * Get runhistory and trajectory from pSMAC run Via the result merger. Searchers rundirs and merges found runhistories and calculates a new trajectory. * Get trajectory * Add regex to requirements * Fixed MO cache problem * Fix MO tests * Fixing MO in AbstractRacer * Accepting ints * Hacking in the sum cost for MO * Fix mypi * Bugfix * Make mypi happy * Fix tests * Make format * Remove num_obj in MO optimizer * A lot of restructuring * Solve importings * Solve importings * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Updated changelog * Still fixing :) * fix docstrings * Change directory of tests * Fix ambiguous variable * Del newlines * fix mypy * mypy and doc fix * Removed int type checking * fix tests * Always return List of incs * fix path issue in example * maint boing facade * Better docstring * Return values if no bounds passed * Removed import * Return only one incumbent * Rename n_optimizers -> n_workers * Update example * Update docstring * Remove legacy function * Update to new signature * Update changelog.md * Format black * Update test * Please pre-commit * Merge development * Fix tests * Fix typos * Format black * Updated text * Increased version * Improved readme * SMAC logo 50% * SMAC logo 50% * Removed dev-2.0 * Added doc workflow for latest tag * Fixing docs latest tag * Fixing docs latest tag * First steps new intensifier * Make it work again * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * Test issues * FIX formatting * Link to old documentation * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * FIX formatting * Fix unit tests * Add annotation * corrected log prob for tophat prior * Removed configurations from callback * Fixed tests * Fix mypy * Save docs as latest version too * Enabled all tests again * Make version an env * Added last version to docs * Global variable fix * Added v before version number * New Intensifier Implementation (#888) - Completely reimplemented the intensifiers (including Successive Halving and Hyperband): All intensifiers support multi-fidelity, multi-objective and multi-threading by nature now. - Expected behaviour for ask-and-tell interface ensured (also for Successive Halving). - Continuing a run is now fully supported. - Added more examples. - Updated documentation based on new implementation. * First benchmark steps * Added a warning * Set xi to 0.01 * Incorporated walltime in trajectory * Working version of benchmark * Changed requirements * Changed xi back to 0.0 again * No duplicate budgets * Fixed bug which updated tracker wrongly * Bugfix: Delete correct bracket+stage * Fixed tests * Added costs to the trajectory * Changed initial design max ratio to 0.25 * Make format * Added more details to intensifier * Finished benchmark * Added benchmark source files * Fixed tests because of initial design changes * Improved runhistory API * Fixed typo (#894) * Merge doc fixes * Doc fixes (#893) Co-authored-by: timruhkopf Co-authored-by: Carolin Benjamins Co-authored-by: dengdifan Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs * Make format * Satisfied mypy * Enabled more tests * Increased version * Mypy * Mypy * Mypy * Fixed critical bug when using highest budget for inc selection * Added SH+HB comparison * Stuff * Updated report * Fixed docstring * Fixed image path * Removed vscode folder * Bugfixes * Doc fixes * Updated changelog * Make isort * Fix typos * Removed new line * Added alpha2 to docs selection * update description * Fix 531 correctly create integers in initial design This PR fixes a bug in which integer hyperparameters would not be created correctly in the initial design. This went unnoticed, except in the case where it was part of a condition and the function deactivate_inactive_hyperparameters was called. This PR fixes this bug by mapping the random float that was sampled by, for example and LHD or Sobol sequence, to the float representation of an integer by calling _transform and _inverse_transform of the integer hyperparameter. * Add contact paragraph to README. * Increase days until stale issues are closed by stalebot (#904) * Edit changelog * Add issue number * Closes #907 * Clarify origin of configurations (#908) * Random forest speed up (#903) Co-authored-by: René Sass * Add: workflow to update pre-commit versions (#874) Co-authored-by: René Sass * Add Git-Flow Workflow Specification * [DOC] acquisition function maximizers * Add stale reminder email action * Increase version number of ConfigSpace * Add guideline for updating copyright notice years to CONTRIBUTING.md * Update requirement for pyrfr (#920) * Update pre-commit: isort,flake8 version numbers * Format black * Fix mypy * Ac bench (#929) * Add Branin AC Task * Add ac optimization type * Nicer plotting * Add ac mode * Update ac task * Make new smac env name adjustable * Increase python version for smac 1.4 * Update benchmark * Update plotting * Goodby comments * Fix path * Fix 1.4 wrapper * Update benchmark results --------- Co-authored-by: Carolin Benjamins * [DOC] added documentation on the incumbent update * Feature/add version to make publish (#931) * Add version, correct package name and use bash in makefile * Increase version * Increase version in makefile, add places with versions to make publish * Fix typo * Transform integer feature names to strings (#930) * Transform int features to str * Fix pre-commit * Update version to 2.0.0 * Update version to 2.0.0 * Update CHANGELOG.md * Add information on deepcave to the readme * Correct doc version * fix(runhistory): set id of the sampled config (#951) * fix: Set config_id of config in runhistory * ci: Set dependabot to target development branch (#963) * feat: Option preventing SMAC setting up logging (#947) * Version 2.0.0 (#898) * Update runhistory2epm.py * pre_commit * Updated changelog * Fixed comment * Updated changelog * Some small improvements * Removed trailing space * Updated changelog * Restricted pynisher * Updated changelog * Link JMLR paper * Update bibtex entry Before: arxiv paper, now journal article. * Fixed citation in docs * Fixed callback typos * BOinG for mixed hyperparameter search space * fix format * maint * maint documentation * maint pre-commit * fix workflow * maint * Add dependabot for workflow files * Bump actions/setup-python from 2 to 4 (#849) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/checkout from 2 to 3 (#850) Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * maint * Updated stale bot * Updated stale options again * Fixed typing * Increased version * rename check_point_in_ss to check_subspace_point * Make format * Uncomment test procedure * docstrings for boing facade * avoid typing namespace everywhere in epm_configuration_chooser * fix docstring in rh2epm_boing * Limit on dask * Trying to fix tests * Replaced typing in all files * docstring for boing subspace * Changed feature requests to feature * allow acq_optimizer_local_kwargs to be passed to boing subspace without acq_optimizer_local * fix pre-commit * Added any * Update facade * Fix typing * Match arguments from pSMAC with SMAC4AC signature * Add description of pSMAC to docstring * Add option to pass facade class to pSMAC * Add example for pSMAC * Removed docstring in readme's example * Corrected text * move construct_gp_kernels to smac.epm_gpytorch.boing_kernels * maint * fix wrongly imported func name in tests * Add reference for pSMAC #609 * Rename example file * Added roadmap to readme * Prettification * Add result merger * Reformat with black * Delete unnecessary import * Fix typing * Inject first trajectory entry from file from first rundir * Add test for ResultMerger * Fix docstring * Get runhistory and trajectory from pSMAC run Via the result merger. Searchers rundirs and merges found runhistories and calculates a new trajectory. * Get trajectory * Add regex to requirements * Fixed MO cache problem * Fix MO tests * Fixing MO in AbstractRacer * Accepting ints * Hacking in the sum cost for MO * Fix mypi * Bugfix * Make mypi happy * Fix tests * Make format * Remove num_obj in MO optimizer * A lot of restructuring * Solve importings * Solve importings * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Updated changelog * Still fixing :) * fix docstrings * Change directory of tests * Fix ambiguous variable * Del newlines * fix mypy * mypy and doc fix * Removed int type checking * fix tests * Always return List of incs * fix path issue in example * maint boing facade * Better docstring * Return values if no bounds passed * Removed import * Return only one incumbent * Rename n_optimizers -> n_workers * Update example * Update docstring * Remove legacy function * Update to new signature * Update changelog.md * Format black * Update test * Please pre-commit * Merge development * Fix tests * Fix typos * Format black * Updated text * Increased version * Improved readme * SMAC logo 50% * SMAC logo 50% * Removed dev-2.0 * Added doc workflow for latest tag * Fixing docs latest tag * Fixing docs latest tag * First steps new intensifier * Make it work again * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * Test issues * FIX formatting * Link to old documentation * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * FIX formatting * Fix unit tests * Add annotation * corrected log prob for tophat prior * Removed configurations from callback * Fixed tests * Fix mypy * Save docs as latest version too * Enabled all tests again * Make version an env * Added last version to docs * Global variable fix * Added v before version number * New Intensifier Implementation (#888) - Completely reimplemented the intensifiers (including Successive Halving and Hyperband): All intensifiers support multi-fidelity, multi-objective and multi-threading by nature now. - Expected behaviour for ask-and-tell interface ensured (also for Successive Halving). - Continuing a run is now fully supported. - Added more examples. - Updated documentation based on new implementation. * First benchmark steps * Added a warning * Set xi to 0.01 * Incorporated walltime in trajectory * Working version of benchmark * Changed requirements * Changed xi back to 0.0 again * No duplicate budgets * Fixed bug which updated tracker wrongly * Bugfix: Delete correct bracket+stage * Fixed tests * Added costs to the trajectory * Changed initial design max ratio to 0.25 * Make format * Added more details to intensifier * Finished benchmark * Added benchmark source files * Fixed tests because of initial design changes * Improved runhistory API * Fixed typo (#894) * Merge doc fixes * Doc fixes (#893) Co-authored-by: timruhkopf Co-authored-by: Carolin Benjamins Co-authored-by: dengdifan Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs * Make format * Satisfied mypy * Enabled more tests * Increased version * Mypy * Mypy * Mypy * Fixed critical bug when using highest budget for inc selection * Added SH+HB comparison * Stuff * Updated report * Fixed docstring * Fixed image path * Removed vscode folder * Bugfixes * Doc fixes * Updated changelog * Make isort * Fix typos * Removed new line * Added alpha2 to docs selection * update description * Fix 531 correctly create integers in initial design This PR fixes a bug in which integer hyperparameters would not be created correctly in the initial design. This went unnoticed, except in the case where it was part of a condition and the function deactivate_inactive_hyperparameters was called. This PR fixes this bug by mapping the random float that was sampled by, for example and LHD or Sobol sequence, to the float representation of an integer by calling _transform and _inverse_transform of the integer hyperparameter. * Add contact paragraph to README. * Increase days until stale issues are closed by stalebot (#904) * Edit changelog * Add issue number * Closes #907 * Clarify origin of configurations (#908) * Random forest speed up (#903) Co-authored-by: René Sass * Add: workflow to update pre-commit versions (#874) Co-authored-by: René Sass * Add Git-Flow Workflow Specification * [DOC] acquisition function maximizers * Add stale reminder email action * Increase version number of ConfigSpace * Add guideline for updating copyright notice years to CONTRIBUTING.md * Update requirement for pyrfr (#920) * Update pre-commit: isort,flake8 version numbers * Format black * Fix mypy * Ac bench (#929) * Add Branin AC Task * Add ac optimization type * Nicer plotting * Add ac mode * Update ac task * Make new smac env name adjustable * Increase python version for smac 1.4 * Update benchmark * Update plotting * Goodby comments * Fix path * Fix 1.4 wrapper * Update benchmark results --------- Co-authored-by: Carolin Benjamins * [DOC] added documentation on the incumbent update * Feature/add version to make publish (#931) * Add version, correct package name and use bash in makefile * Increase version * Increase version in makefile, add places with versions to make publish * Fix typo * Transform integer feature names to strings (#930) * Transform int features to str * Fix pre-commit * Update version to 2.0.0 * Update version to 2.0.0 * Update CHANGELOG.md * Add information on deepcave to the readme * Correct doc version --------- Co-authored-by: dengdifan <33290713+dengdifan@users.noreply.github.com> Co-authored-by: dengdifan Co-authored-by: Matthias Feurer Co-authored-by: Carolin Benjamins Co-authored-by: eddiebergman Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Tim Ruhkopf * feat: Option preventing SMAC setting up logging * fix: Run isort * Bump peter-evans/create-pull-request from 3 to 5 (#958) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 3 to 5. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v3...v5) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/setup-python from 2 to 4 (#945) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat: Option preventing SMAC setting up logging * fix: Run isort * fix: Add to changelog * feat: Option preventing SMAC setting up logging * fix: Run isort --------- Signed-off-by: dependabot[bot] Co-authored-by: René Sass Co-authored-by: dengdifan <33290713+dengdifan@users.noreply.github.com> Co-authored-by: dengdifan Co-authored-by: Matthias Feurer Co-authored-by: Carolin Benjamins Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Tim Ruhkopf * Citation update (#961) * update CITATION.cff * update order in CITATION.cff * fix citation date * fix citation * maint * Adjust hyperband configuration distribution across brackets * Compute the actual differences between the isb keys. (#957) * Compute the actual differences between the isb keys. * Added change to CHANGELOG.md * Adjust schedule for stale reminder * fix(logging): Prevent automatic logging setup at init (#970) * fix(logging): Prevent automatic logging setup at init * fix(logging): Rely on `setup_logging` to handle arg * Fix validate method of smbo, update docstring. * Allow callbacks to be added to a specific index, make callback registering public * Update CHANGELOG.md * Fix broken references (#988) * Adapt developer install instructions to include pre-commit installation (#994) * [Feature] Initial Design now supports executing the default config provided by the user by means of the configspace he generated. (and is no longer ignored) * Pipe dask_client through facade for exec on hpc (#983) * Pipe dask_client through facade for exec on hpc * Handle pynisher's memory arg Before: Could only pass int via SMAC with the default unit B, but now tuple/list is possible * Update CHANGELOG.md * Make format * No printi --------- Co-authored-by: Carolin Benjamins * [Bug-fix] Pr was failing due to mutable additional configs default * [doc-fix] the count of initial design configs is computed slightly differently * Feature/metadata callback (#999) * Add example for using a callback to log run metadata to a file * Add issue number in changelog * Changelog formatting * Simplify metadata example * Simplify metadata example * Put callback in smac directory to allow importing it in custom repos * Move callback files to callback directory * Edit changelog * Correct metadata callback argument types * Additional metadata arguments in example * Set metadata default values in callback * Edit comment to fix PR * Documentation/dask client example (#1001) * Update requirements (dask_jobqueue) * Fix broken link * Ignore parallel example * Add parallelization example * Remove imports, add comment abt dask client * Add E's information * Update docs * Add comment about n_workers --------- Co-authored-by: Carolin Benjamins * Documentation/dask client example (#1001) * Update requirements (dask_jobqueue) * Fix broken link * Ignore parallel example * Add parallelization example * Remove imports, add comment abt dask client * Add E's information * Update docs * Add comment about n_workers --------- Co-authored-by: Carolin Benjamins * Feature/dask client (#1002) * Add info message that scenario.n_workers is ignored When passing a dask client, directly set n_workers in client. * Change info to warning --------- Co-authored-by: Carolin Benjamins * Update smac version (#1003) * Update smac version * Add hint on github release in make publish * Update copyright year * check if config in rh when storing state (#997) * check if config in rh when storing state * maint * remove unnecessary changes * Adapt developer install instructions to include pre-commit installation (#994) * Pipe dask_client through facade for exec on hpc (#983) * Pipe dask_client through facade for exec on hpc * Handle pynisher's memory arg Before: Could only pass int via SMAC with the default unit B, but now tuple/list is possible * Update CHANGELOG.md * Make format * No printi --------- Co-authored-by: Carolin Benjamins * update changelog * Increase tolerance for runhistory restoration test --------- Co-authored-by: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: Helena Graf * Add a workaround to be able to pass a dataset via dask.scatter (#993) * [feat] Support dask.scatter * [fix] Add shared_data arg in abstract_facade * Address the comments by Eddie * [fix] Fix errors raised by Eddie * [format] Apply black locally * [fix] Get retries back to 16 as it was a mistake * [doc] Update CHANGELOG --------- Co-authored-by: helegraf Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> * Adding Binder links for example (#976) * binder * maint * fix dependencies * maint docs * Revert "maint docs" This reverts commit 4fa1a0fae8dfa0b646e43273d50f72ddd76bf191. * add requirement --------- Co-authored-by: Difan Deng * Remove commit message from dependabot PRs (#1009) --------- Signed-off-by: dependabot[bot] Co-authored-by: dengdifan Co-authored-by: René Sass Co-authored-by: Carolin Benjamins Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Matthias Feurer Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Eddie Bergman Co-authored-by: Tim Ruhkopf Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> Co-authored-by: Aron Bahram Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Co-authored-by: Difan Deng * Fix broken link --------- Signed-off-by: dependabot[bot] Co-authored-by: dengdifan Co-authored-by: René Sass Co-authored-by: Carolin Benjamins Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Matthias Feurer Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Eddie Bergman Co-authored-by: Tim Ruhkopf Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> Co-authored-by: Aron Bahram Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Co-authored-by: Difan Deng * Drop torch requirements (#1015) * Remove torch requirements * Fix format --------- Co-authored-by: Carolin Benjamins * Remove leftover CLI entry points (#1014) * Fix incumbent selection in case of SH+MO * Update CHANGELOG.md * Feature/GitHub actions pr draft (#1008) Update the workflows only to trigger extensive tests for PRs that are not in the draft stage --------- Co-authored-by: timruhkopf Co-authored-by: Helena Graf * Update stale report to include issues with recent interactions (#1025) * Add recently updated issues to list of reminders. * Add recently updated issues to list of reminders. * Add recently updated issues to list of reminders. * Raise error if resource limitation and parallelization is requested (#1023) * Add warnings for parallelism doc * Check for parallelization + pynisher Does not work together, throw error. Does not work bc Dask pickles jobs and pynisher runs the target function in a process. * Fix whitespaces --------- Co-authored-by: Carolin Benjamins * Raise an error for invalid scatter data (#1018) * [enhance] Add an error for scatter data * Update ChangeLog --------- Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> * fix(configspace): shows all keys in print (#1043) * Fix callback order (#1040) Co-authored-by: Carolin Benjamins * fix: dtypes (#1044) * Fix(config_selector): Budget checking (#1039) * Fix: target runner with partial func (#1045) * Feature/improve documentation (#1028) * Experimental instruction for installing SMAC in windows via WSL * More detailed documentation regarding continuing runs * Fixes * Fixes * Fixes * Update docs/10_experimental.rst Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> * Fixes from PR * Fixes from PR * Fixes from PR * Fixes from PR * Fixes from PR --------- Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> * Handle configspace as dictionary in mlp example (#1057) * Handle configspace as dictionary in mlp example * Adapt sgd loss to newest scikit-learn version * Add change to changelog * Handle configspace as dictionary in parego example * Correct get usage * Raise version number * Add missing period in docs. * allow rf to impute OrdinalHyperparameter (#1065) * allow rf to impute OrdinalHyperparameter * Add missing period in docs. * Add issue number in changelog. --------- Co-authored-by: Sarah Krebs * fix dask_scheduler_file path (#1055) * fix dask_scheduler_file path * update change log --------- Co-authored-by: dengdifan * Fix config rejection for #1068 (#1069) * racing: make sure config only deleted from rejected configs if among incumbents upon update * test case for incumbent rejection * update code comments * added missing newline ahead of new test case * update change log --------- Co-authored-by: dengdifan Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> * fix(RandomForest): Always cast `seed` to `int` (#1084) * fix(RandomForest): Always cast `seed` to `int` In cases where seeds were generated from some numpy objects, sometimes you'd get back an `np.integer`, which causes the program to crash when communicating with `pyrfr` through `swig`. It seems that swig doesn't know that it's an _int-like_ and so we explicitly cast it to `int`. * Update CHANGELOG.md --------- Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> * Bump actions/checkout from 3 to 4 (#1072) * Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update CHANGELOG.md --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: benjamc * chore: log warn on condition when custom dask client is provided (#1071) * Bump conda-incubator/setup-miniconda from 2 to 3 (#1087) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 2 to 3. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v2...v3) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Propagate the Scenario random seed to get_random_design (#1066) * Propagate the Scenario's random seed to get_random_design * add changelog line --------- Co-authored-by: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> * [#1056] Add example on intensify for cross-validation. (#1061) * Add example on intensify for cross-validation. * Improve example * Address Difan's comment * Update CHANGELOG.md * Change stale-reminder to recent-reminder (#1096) * Change stale-reminder to recent-reminder (#1095) (#1099) * Dont use mutable lists as default args (#1097) * Don't use mutable default argument. If this isn't respected, then additional_configs can be built up overtime when multiple optimizers are created in a single process. This can occur when the use_default_config option is set True and/or when additional_configs are provided. When different ConfigSpaces are provided this can result in an error due to mismatched defaults, for instance. In other cases it can result in non-deterministic behavior due to the random ordering of the configs that get added (e.g., with different parallel pytest runs). - [ ] Cleanup other sources of this error as well elsewhere in the code. - [ ] Add pylint checks for this issue. * fixup some others mutable defaults * add pylint checks * fix more related pylint issues * add changelog line * Update pre-commit config * Update pre-commit config * Remove TODO formulation as an issue already exists for this --------- Co-authored-by: Brian Kroth * Bump actions/setup-python from 4 to 5 (#1089) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump peter-evans/create-pull-request from 5 to 6 (#1093) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 5 to 6. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v5...v6) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * test: Fix upper bounds for `test_transformer()` and no random in `test_transformer_conditional()` (#1102) * test: Don't rely on randomization for the test * test: Include import * test: Fix boundaries for catgoricals in test * test: Missing import * doc: Make todo not more clear about when to upgrade * Change stale-reminder to recent-reminder (#1095) (#1107) * Fix typo in docstring for `MultiFideltyFacade.get_initial_design` (#1104) * Fix typo in docstring for `MultiFideltyFacade.get_initial_design` The docstring specifies 0.1 is used as a default value for max_ratio, however in code 0.25 is used. * Bug/acquisition maximizer sampling (#1106) * Adapt how n_points works in the config_selector.py Also remove LocalAndSortedPriorRandomSearch by integrating it into LocalAndSortedRandomSearch * Change retrain_after to 1 for BlackBOx Facade * Fix bugs in local_and_random_search.py * Edit CHANGELOG.md * fix acq maximizer docstring * call get config selector from super class * Update random and local search to new functionality * Update CHANGELOG.md * Fix sampling of initial points for local search when less previous configs than n_points * Re-format blackbox_facade.py --------- Co-authored-by: Helena Graf Co-authored-by: dengdifan * Update version number to 2.1.0 --------- Signed-off-by: dependabot[bot] Co-authored-by: René Sass Co-authored-by: dengdifan Co-authored-by: Carolin Benjamins Co-authored-by: Katharina Eggensperger Co-authored-by: Matthias Feurer Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Eddie Bergman Co-authored-by: Tim Ruhkopf Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> Co-authored-by: Aron Bahram Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Co-authored-by: Difan Deng Co-authored-by: Bastian Zimmermann <10774221+BastianZim@users.noreply.github.com> Co-authored-by: Weihuang Wen Co-authored-by: PhilippBordne <71140732+PhilippBordne@users.noreply.github.com> Co-authored-by: SimonThormeyer <49559340+SimonThormeyer@users.noreply.github.com> Co-authored-by: Brian Kroth Co-authored-by: Brian Kroth Co-authored-by: fleance --- .github/workflows/citation.yml | 2 +- .github/workflows/dist.yml | 4 +- .github/workflows/docs.yml | 4 +- .github/workflows/pre-commit-update.yml | 6 +- .github/workflows/pre-commit.yml | 5 +- .github/workflows/pytest.yml | 12 +- .pre-commit-config.yaml | 13 + CHANGELOG.md | 30 ++- CITATION.cff | 2 +- Makefile | 2 +- benchmark/src/wrappers/v20.py | 2 +- .../4_intensify_crossvalidation.py | 127 ++++++++++ pyproject.toml | 5 + setup.py | 1 + smac/__init__.py | 2 +- smac/acquisition/maximizer/__init__.py | 2 - .../abstract_acqusition_maximizer.py | 13 +- .../maximizer/differential_evolution.py | 1 + smac/acquisition/maximizer/helpers.py | 47 +--- .../maximizer/local_and_random_search.py | 231 +++++++----------- smac/acquisition/maximizer/local_search.py | 29 +-- smac/facade/abstract_facade.py | 7 +- smac/facade/algorithm_configuration_facade.py | 4 +- smac/facade/blackbox_facade.py | 17 +- .../hyperparameter_optimization_facade.py | 4 +- smac/facade/multi_fidelity_facade.py | 6 +- smac/facade/random_facade.py | 4 +- smac/intensifier/abstract_intensifier.py | 8 +- smac/main/config_selector.py | 7 +- .../random_forest/abstract_random_forest.py | 5 +- smac/model/random_forest/random_forest.py | 4 +- smac/runhistory/encoder/abstract_encoder.py | 16 +- smac/runhistory/runhistory.py | 4 +- smac/runner/abstract_runner.py | 4 +- smac/runner/dask_runner.py | 2 +- smac/runner/target_function_runner.py | 4 +- smac/runner/target_function_script_runner.py | 4 +- tests/test_acquisition/test_maximizers.py | 28 +-- .../test_abstract_intensifier.py | 42 ++++ tests/test_model/test_rf.py | 14 +- .../test_runhistory_encoder.py | 138 ++++++++--- 41 files changed, 552 insertions(+), 310 deletions(-) create mode 100644 examples/4_advanced_optimizer/4_intensify_crossvalidation.py diff --git a/.github/workflows/citation.yml b/.github/workflows/citation.yml index 9ca4ace4b..91509b900 100644 --- a/.github/workflows/citation.yml +++ b/.github/workflows/citation.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out a copy of the repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Check whether the citation metadata from CITATION.cff is valid uses: citation-file-format/cffconvert-github-action@2.0.0 diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index 0965ea2f1..8e9908060 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -28,10 +28,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d89791712..5f03e854f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -32,10 +32,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" diff --git a/.github/workflows/pre-commit-update.yml b/.github/workflows/pre-commit-update.yml index 1688f60d7..b027befb2 100644 --- a/.github/workflows/pre-commit-update.yml +++ b/.github/workflows/pre-commit-update.yml @@ -11,9 +11,9 @@ jobs: auto-update: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 - uses: browniebroke/pre-commit-autoupdate-action@main @@ -21,7 +21,7 @@ jobs: run: | pre-commit run --all-files - - uses: peter-evans/create-pull-request@v5 + - uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.GITHUB_TOKEN }} branch: update/pre-commit-hooks diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index e9ba1a01b..31d957762 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -27,12 +27,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: recursive - name: Setup Python 3.10 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" @@ -40,6 +40,7 @@ jobs: run: | pip install pre-commit pre-commit install + pip install pylint - name: Run pre-commit run: | diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 37e15a472..c94e246bb 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -54,10 +54,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -108,10 +108,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Conda install - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true python-version: ${{ matrix.python-version }} @@ -148,10 +148,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dda7e5cae..6b2d3934f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,3 +43,16 @@ repos: name: flake8 smac files: smac exclude: "scripts|tests" + + - repo: local + hooks: + - id: pylint + name: pylint + entry: pylint + language: system + types: [python] + args: + [ + "-rn", # Only display messages + "-sn", # Don't display the score + ] diff --git a/CHANGELOG.md b/CHANGELOG.md index dbcac80e4..f6768b689 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,37 @@ +# 2.1.0 + +## Improvements +- Change the surrogate model to be retrained after every iteration by default in the case of blackbox optimization + (#1106). +- Integrate `LocalAndSortedPriorRandomSearch` functionality into `LocalAndSortedRandomSearch` (#1106). +- Change the way the `LocalAndSortedRandomSearch` works such that the incumbent always is a starting point and that + random configurations are sampled as the basis of the local search, not in addition (#1106). + +## Bugfixes +- Fix path for dask scheduler file (#1055). +- Add OrdinalHyperparameter for random forest imputer (#1065). +- Don't use mutable default argument (#1067). +- Propagate the Scenario random seed to `get_random_design` (#1066). +- Configurations that fail to become incumbents will be added to the rejected lists (#1069). +- SMAC RandomForest doesn't crash when `np.integer` used, i.e. as generated from a `np.random.RandomState` (#1084). +- Fix the handling of n_points/ challengers in the acquisition maximizers, such that this number now functions as the + number of points that are sampled from the acquisition function to find the next challengers. Now also doesn't + restrict the config selector to n_retrain many points for finding the max, and instead uses the defaults that are + defined via facades/ scenarios (#1106). + +## Misc +- ci: Update action version (#1072). + +## Minor +- When a custom dask client is provided, emit the warning that the `n_workers` parameter is ignored only if it deviates from its default value, `1` ([#1071](https://github.com/automl/SMAC3/pull/1071)). + # 2.0.2 ## Improvements -- Add an error when we get an empty dict data_to_scatter so that we can avoid an internal error caused in Dask precautiously +- Add an error when we get an empty dict data_to_scatter so that we can avoid an internal error caused in Dask precautiously. - Add experimental instruction for installing SMAC in Windows via a WSL. - More detailed documentation regarding continuing runs. +- Add a new example that demonstrates the use of intensification to speed up cross-validation for machine learning. ## Bugfixes - Fix bug in the incumbent selection in the case that multi-fidelity is combined with multi-objective (#1019). diff --git a/CITATION.cff b/CITATION.cff index a984b24c5..ece8325c6 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,7 +9,7 @@ date-released: "2016-08-17" url: "https://automl.github.io/SMAC3/master/index.html" repository-code: "https://github.com/automl/SMAC3" -version: "2.0.2" +version: "2.1.0" type: "software" keywords: diff --git a/Makefile b/Makefile index ba7a4268b..195773082 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ SHELL := /bin/bash NAME := SMAC3 PACKAGE_NAME := smac -VERSION := 2.0.2 +VERSION := 2.1.0 DIR := "${CURDIR}" SOURCE_DIR := ${PACKAGE_NAME} diff --git a/benchmark/src/wrappers/v20.py b/benchmark/src/wrappers/v20.py index 9f2be56dc..3ee376a37 100644 --- a/benchmark/src/wrappers/v20.py +++ b/benchmark/src/wrappers/v20.py @@ -6,7 +6,7 @@ class Version20(Wrapper): - supported_versions: list[str] = ["2.0.2"] + supported_versions: list[str] = ["2.1.0"] def __init__(self, task: Task, seed: int) -> None: super().__init__(task, seed) diff --git a/examples/4_advanced_optimizer/4_intensify_crossvalidation.py b/examples/4_advanced_optimizer/4_intensify_crossvalidation.py new file mode 100644 index 000000000..d215dd8ec --- /dev/null +++ b/examples/4_advanced_optimizer/4_intensify_crossvalidation.py @@ -0,0 +1,127 @@ +""" +Speeding up Cross-Validation with Intensification +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +An example of optimizing a simple support vector machine on the digits dataset. In contrast to the +[simple example](examples/1_basics/2_svm_cv.py), in which all cross-validation folds are executed +at once, we use the intensification mechanism described in the original +[SMAC paper](https://link.springer.com/chapter/10.1007/978-3-642-25566-3_40) as also demonstrated +by [Auto-WEKA](https://dl.acm.org/doi/10.1145/2487575.2487629). This mechanism allows us to +terminate the evaluation of a configuration if after a certain number of folds, the configuration +is found to be worse than the incumbent configuration. This is especially useful if the evaluation +of a configuration is expensive, e.g., if we have to train a neural network or if we have to +evaluate the configuration on a large dataset. +""" +__copyright__ = "Copyright 2023, AutoML.org Freiburg-Hannover" +__license__ = "3-clause BSD" + +N_FOLDS = 10 # Global variable that determines the number of folds + +from ConfigSpace import Configuration, ConfigurationSpace, Float +from sklearn import datasets, svm +from sklearn.model_selection import StratifiedKFold + +from smac import HyperparameterOptimizationFacade, Scenario +from smac.intensifier import Intensifier + +# We load the digits dataset, a small-scale 10-class digit recognition dataset +X, y = datasets.load_digits(return_X_y=True) + + +class SVM: + @property + def configspace(self) -> ConfigurationSpace: + # Build Configuration Space which defines all parameters and their ranges + cs = ConfigurationSpace(seed=0) + + # First we create our hyperparameters + C = Float("C", (2 ** - 5, 2 ** 15), default=1.0, log=True) + gamma = Float("gamma", (2 ** -15, 2 ** 3), default=1.0, log=True) + + # Add hyperparameters to our configspace + cs.add_hyperparameters([C, gamma]) + + return cs + + def train(self, config: Configuration, instance: str, seed: int = 0) -> float: + """Creates a SVM based on a configuration and evaluate on the given fold of the digits dataset + + Parameters + ---------- + config: Configuration + The configuration to train the SVM. + instance: str + The name of the instance this configuration should be evaluated on. This is always of type + string by definition. In our case we cast to int, but this could also be the filename of a + problem instance to be loaded. + seed: int + The seed used for this call. + """ + instance = int(instance) + classifier = svm.SVC(**config, random_state=seed) + splitter = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=seed) + for k, (train_idx, test_idx) in enumerate(splitter.split(X=X, y=y)): + if k != instance: + continue + else: + train_X = X[train_idx] + train_y = y[train_idx] + test_X = X[test_idx] + test_y = y[test_idx] + classifier.fit(train_X, train_y) + cost = 1 - classifier.score(test_X, test_y) + + return cost + + +if __name__ == "__main__": + classifier = SVM() + + # Next, we create an object, holding general information about the run + scenario = Scenario( + classifier.configspace, + n_trials=50, # We want to run max 50 trials (combination of config and instances in the case of + # deterministic=True. In the case of deterministic=False, this would be the + # combination of instances, seeds and configs). The number of distinct configurations + # evaluated by SMAC will be lower than this number because some of the configurations + # will be executed on more than one instance (CV fold). + instances=[f"{i}" for i in range(N_FOLDS)], # Specify all instances by their name (as a string) + instance_features={f"{i}": [i] for i in range(N_FOLDS)}, # breaks SMAC + deterministic=True # To simplify the problem we make SMAC believe that we have a deterministic + # optimization problem. + + ) + + # We want to run the facade's default initial design, but we want to change the number + # of initial configs to 5. + initial_design = HyperparameterOptimizationFacade.get_initial_design(scenario, n_configs=5) + + # Now we use SMAC to find the best hyperparameters + smac = HyperparameterOptimizationFacade( + scenario, + classifier.train, + initial_design=initial_design, + overwrite=True, # If the run exists, we overwrite it; alternatively, we can continue from last state + # The next line defines the intensifier, i.e., the module that governs the selection of + # instance-seed pairs. Since we set deterministic to True above, it only governs the instance in + # this example. Technically, it is not necessary to create the intensifier as a user, but it is + # necessary to do so because we change the argument max_config_calls (the number of instance-seed pairs + # per configuration to try) to the number of cross-validation folds, while the default would be 3. + intensifier=Intensifier(scenario=scenario, max_config_calls=N_FOLDS, seed=0) + ) + + incumbent = smac.optimize() + + # Get cost of default configuration + default_cost = smac.validate(classifier.configspace.get_default_configuration()) + print(f"Default cost: {default_cost}") + + # Let's calculate the cost of the incumbent + incumbent_cost = smac.validate(incumbent) + print(f"Incumbent cost: {incumbent_cost}") + + # Let's see how many configurations we have evaluated. If this number is higher than 5, we have looked + # at more configurations than would have been possible with regular cross-validation, where the number + # of configurations would be determined by the number of trials divided by the number of folds (50 / 10). + runhistory = smac.runhistory + print(f"Number of evaluated configurations: {len(runhistory.config_ids)}") \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index c0e1ae4ed..77783d8a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,11 @@ add-ignore = [ # http://www.pydocstyle.org/en/stable/error_codes.html "D415", # First line should end with a period, question mark, or exclamation point ] +[tool.pylint."messages control"] +# FIXME: This is to do a staged introduction of pylint checks for just a single class of problems initially (#1067). +disable = ["all"] +enable = ["dangerous-default-value"] + [tool.mypy] python_version = "3.9" show_error_codes = true diff --git a/setup.py b/setup.py index f194d3677..27ec7e695 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ def read_file(filepath: str) -> str: "pydocstyle", "flake8", "pre-commit", + "pylint", ], } diff --git a/smac/__init__.py b/smac/__init__.py index ab2781297..3241da062 100644 --- a/smac/__init__.py +++ b/smac/__init__.py @@ -19,7 +19,7 @@ Copyright {datetime.date.today().strftime('%Y')}, Marius Lindauer, Katharina Eggensperger, Matthias Feurer, André Biedenkapp, Difan Deng, Carolin Benjamins, Tim Ruhkopf, René Sass and Frank Hutter""" -version = "2.0.2" +version = "2.1.0" try: diff --git a/smac/acquisition/maximizer/__init__.py b/smac/acquisition/maximizer/__init__.py index 6fea59b45..5e3756190 100644 --- a/smac/acquisition/maximizer/__init__.py +++ b/smac/acquisition/maximizer/__init__.py @@ -3,7 +3,6 @@ ) from smac.acquisition.maximizer.differential_evolution import DifferentialEvolution from smac.acquisition.maximizer.local_and_random_search import ( - LocalAndSortedPriorRandomSearch, LocalAndSortedRandomSearch, ) from smac.acquisition.maximizer.local_search import LocalSearch @@ -13,7 +12,6 @@ "AbstractAcquisitionMaximizer", "DifferentialEvolution", "LocalAndSortedRandomSearch", - "LocalAndSortedPriorRandomSearch", "LocalSearch", "RandomSearch", ] diff --git a/smac/acquisition/maximizer/abstract_acqusition_maximizer.py b/smac/acquisition/maximizer/abstract_acqusition_maximizer.py index 5a14a8814..e148cb6ed 100644 --- a/smac/acquisition/maximizer/abstract_acqusition_maximizer.py +++ b/smac/acquisition/maximizer/abstract_acqusition_maximizer.py @@ -27,12 +27,9 @@ class AbstractAcquisitionMaximizer: Parameters ---------- - configspace : ConfigurationSpace - acquisition_function : AbstractAcquisitionFunction - challengers : int, defaults to 5000 - Number of configurations to sample from the configuration space to get - the acquisition function value for, thus challenging the current - incumbent and becoming a candidate for the next function evaluation. + configspace : ConfigurationSpace acquisition_function : AbstractAcquisitionFunction + challengers : int, defaults to 5000 Number of configurations sampled during the optimization process, + details depend on the used maximizer. Also, the number of configurations that is returned by calling `maximize`. seed : int, defaults to 0 """ @@ -85,8 +82,8 @@ def maximize( previous_configs: list[Configuration] Previous evaluated configurations. n_points: int, defaults to None - Number of points to be sampled. If `n_points` is not specified, - `self._challengers` is used. + Number of points to be sampled & number of configurations to be returned. If `n_points` is not specified, + `self._challengers` is used. Semantics depend on concrete implementation. random_design: AbstractRandomDesign, defaults to None Part of the returned ChallengerList such that we can interleave random configurations by a scheme defined by the random design. The method `random_design.next_iteration()` diff --git a/smac/acquisition/maximizer/differential_evolution.py b/smac/acquisition/maximizer/differential_evolution.py index d1201c19c..0f2ce15e6 100644 --- a/smac/acquisition/maximizer/differential_evolution.py +++ b/smac/acquisition/maximizer/differential_evolution.py @@ -30,6 +30,7 @@ def _maximize( previous_configs: list[Configuration], n_points: int, ) -> list[tuple[float, Configuration]]: + # n_points is not used here, but is required by the interface configs: list[tuple[float, Configuration]] = [] diff --git a/smac/acquisition/maximizer/helpers.py b/smac/acquisition/maximizer/helpers.py index d7319f20f..db06e2aa4 100644 --- a/smac/acquisition/maximizer/helpers.py +++ b/smac/acquisition/maximizer/helpers.py @@ -4,8 +4,8 @@ from ConfigSpace import Configuration, ConfigurationSpace +from smac.random_design import ProbabilityRandomDesign from smac.random_design.abstract_random_design import AbstractRandomDesign -from smac.random_design.modulus_design import ModulusRandomDesign class ChallengerList(Iterator): @@ -20,7 +20,7 @@ class ChallengerList(Iterator): ---------- configspace : ConfigurationSpace challenger_callback : Callable - Callback function which returns a list of challengers (without interleaved random configurations, must a be a + Callback function which returns a list of challengers (without interleaved random configurations), must a be a python closure. random_design : AbstractRandomDesign | None, defaults to ModulusRandomDesign(modulus=2.0) Which random design should be used. @@ -30,7 +30,7 @@ def __init__( self, configspace: ConfigurationSpace, challenger_callback: Callable, - random_design: AbstractRandomDesign | None = ModulusRandomDesign(modulus=2.0), + random_design: AbstractRandomDesign | None = ProbabilityRandomDesign(seed=0, probability=0.08447232371720552), ): self._challengers_callback = challenger_callback self._challengers: list[Configuration] | None = None @@ -72,44 +72,3 @@ def __len__(self) -> int: self._challengers = self._challengers_callback() return len(self._challengers) - self._index - - -''' -class FixedSet(AbstractAcquisitionMaximizer): - def __init__( - self, - configurations: list[Configuration], - acquisition_function: AbstractAcquisitionFunction, - configspace: ConfigurationSpace, - challengers: int = 5000, - seed: int = 0, - ): - """Maximize the acquisition function over a finite list of configurations. - - Parameters - ---------- - configurations : list[~smac._configspace.Configuration] - Candidate configurations - acquisition_function : ~smac.acquisition.AbstractAcquisitionFunction - - configspace : ~smac._configspace.ConfigurationSpace - - rng : np.random.RandomState or int, optional - """ - super().__init__( - acquisition_function=acquisition_function, configspace=configspace, challengers=challengers, seed=seed - ) - self.configurations = configurations - - def _maximize( - self, - runhistory: RunHistory, - stats: Stats, - n_points: int, - ) -> list[tuple[float, Configuration]]: - configurations = copy.deepcopy(self.configurations) - for config in configurations: - config.origin = "Fixed Set" - - return self._sort_by_acquisition_value(configurations) -''' diff --git a/smac/acquisition/maximizer/local_and_random_search.py b/smac/acquisition/maximizer/local_and_random_search.py index bb6daaef5..d81af2f69 100644 --- a/smac/acquisition/maximizer/local_and_random_search.py +++ b/smac/acquisition/maximizer/local_and_random_search.py @@ -21,7 +21,7 @@ class LocalAndSortedRandomSearch(AbstractAcquisitionMaximizer): """Implement SMAC's default acquisition function optimization. - This optimizer performs local search from the previous best points according, to the acquisition + This optimizer performs local search from the previous best points according to the acquisition function, uses the acquisition function to sort randomly sampled configurations. Random configurations are interleaved by the main SMAC code. @@ -31,6 +31,10 @@ class LocalAndSortedRandomSearch(AbstractAcquisitionMaximizer): Parameters ---------- configspace : ConfigurationSpace + uniform_configspace : ConfigurationSpace + A version of the user-defined ConfigurationSpace where all parameters are uniform (or have their weights removed + in the case of a categorical hyperparameter). Can optionally be given and sampling ratios be defined via the + `prior_sampling_fraction` parameter. acquisition_function : AbstractAcquisitionFunction | None, defaults to None challengers : int, defaults to 5000 Number of challengers. @@ -40,6 +44,9 @@ class LocalAndSortedRandomSearch(AbstractAcquisitionMaximizer): [LocalSearch] number of steps during a plateau walk before local search terminates. local_search_iterations: int, defauts to 10 [Local Search] number of local search iterations. + prior_sampling_fraction: float, defaults to 0.5 + The ratio of random samples that are taken from the user-defined ConfigurationSpace, as opposed to the uniform + version (needs `uniform_configspace`to be defined). seed : int, defaults to 0 """ @@ -52,6 +59,8 @@ def __init__( n_steps_plateau_walk: int = 10, local_search_iterations: int = 10, seed: int = 0, + uniform_configspace: ConfigurationSpace | None = None, + prior_sampling_fraction: float | None = None, ) -> None: super().__init__( configspace, @@ -60,11 +69,28 @@ def __init__( seed=seed, ) - self._random_search = RandomSearch( - configspace=configspace, - acquisition_function=acquisition_function, - seed=seed, - ) + if uniform_configspace is not None and prior_sampling_fraction is None: + prior_sampling_fraction = 0.5 + if uniform_configspace is None and prior_sampling_fraction is not None: + raise ValueError("If `prior_sampling_fraction` is given, `uniform_configspace` must be defined.") + if uniform_configspace is not None and prior_sampling_fraction is not None: + self._prior_random_search = RandomSearch( + acquisition_function=acquisition_function, + configspace=configspace, + seed=seed, + ) + + self._uniform_random_search = RandomSearch( + acquisition_function=acquisition_function, + configspace=uniform_configspace, + seed=seed, + ) + else: + self._random_search = RandomSearch( + configspace=configspace, + acquisition_function=acquisition_function, + seed=seed, + ) self._local_search = LocalSearch( configspace=configspace, @@ -75,6 +101,8 @@ def __init__( ) self._local_search_iterations = local_search_iterations + self._prior_sampling_fraction = prior_sampling_fraction + self._uniform_configspace = uniform_configspace @property def acquisition_function(self) -> AbstractAcquisitionFunction | None: # noqa: D102 @@ -84,18 +112,31 @@ def acquisition_function(self) -> AbstractAcquisitionFunction | None: # noqa: D @acquisition_function.setter def acquisition_function(self, acquisition_function: AbstractAcquisitionFunction) -> None: self._acquisition_function = acquisition_function - self._random_search._acquisition_function = acquisition_function + if self._uniform_configspace is not None: + self._prior_random_search._acquisition_function = acquisition_function + self._uniform_random_search._acquisition_function = acquisition_function + else: + self._random_search._acquisition_function = acquisition_function self._local_search._acquisition_function = acquisition_function @property def meta(self) -> dict[str, Any]: # noqa: D102 meta = super().meta - meta.update( - { - "random_search": self._random_search.meta, - "local_search": self._local_search.meta, - } - ) + if self._uniform_configspace is None: + meta.update( + { + "random_search": self._random_search.meta, + "local_search": self._local_search.meta, + } + ) + else: + meta.update( + { + "prior_random_search": self._prior_random_search.meta, + "uniform_random_search": self._uniform_random_search.meta, + "local_search": self._local_search.meta, + } + ) return meta @@ -105,147 +146,45 @@ def _maximize( n_points: int, ) -> list[tuple[float, Configuration]]: - # Get configurations sorted by EI - next_configs_by_random_search_sorted = self._random_search._maximize( - previous_configs=previous_configs, - n_points=n_points, - _sorted=True, - ) - + if self._uniform_configspace is not None and self._prior_sampling_fraction is not None: + # Get configurations sorted by acquisition function value + next_configs_by_prior_random_search_sorted = self._prior_random_search._maximize( + previous_configs, + round(n_points * self._prior_sampling_fraction), + _sorted=True, + ) + + # Get configurations sorted by acquisition function value + next_configs_by_uniform_random_search_sorted = self._uniform_random_search._maximize( + previous_configs, + round(n_points * (1 - self._prior_sampling_fraction)), + _sorted=True, + ) + next_configs_by_random_search_sorted = ( + next_configs_by_uniform_random_search_sorted + next_configs_by_prior_random_search_sorted + ) + next_configs_by_random_search_sorted.sort(reverse=True, key=lambda x: x[0]) + else: + # Get configurations sorted by acquisition function value + next_configs_by_random_search_sorted = self._random_search._maximize( + previous_configs=previous_configs, + n_points=n_points, + _sorted=True, + ) + + # Choose the best self._local_search_iterations random configs to start the local search, and choose only + # incumbent from previous configs + random_starting_points = next_configs_by_random_search_sorted[: self._local_search_iterations] next_configs_by_local_search = self._local_search._maximize( previous_configs=previous_configs, n_points=self._local_search_iterations, - additional_start_points=next_configs_by_random_search_sorted, + additional_start_points=random_starting_points, ) - # Having the configurations from random search, sorted by their - # acquisition function value is important for the first few iterations - # of SMAC. As long as the random forest predicts constant value, we - # want to use only random configurations. Having them at the begging of - # the list ensures this (even after adding the configurations by local - # search, and then sorting them) - next_configs_by_acq_value = next_configs_by_random_search_sorted + next_configs_by_local_search + next_configs_by_acq_value = next_configs_by_local_search next_configs_by_acq_value.sort(reverse=True, key=lambda x: x[0]) first_five = [f"{_[0]} ({_[1].origin})" for _ in next_configs_by_acq_value[:5]] logger.debug(f"First 5 acquisition function values of selected configurations:\n{', '.join(first_five)}") return next_configs_by_acq_value - - -class LocalAndSortedPriorRandomSearch(AbstractAcquisitionMaximizer): - """Implements SMAC's default acquisition function optimization. - - This optimizer performs local search from the previous best points according to the acquisition function, uses the - acquisition function to sort randomly sampled configurations. Random configurations are interleaved by the main SMAC - code. The random configurations are retrieved from two different ConfigurationSpaces - one which uses priors - (e.g. NormalFloatHP) and is defined by the user, and one that is a uniform version of the same space, i.e. with the - priors removed. - - Parameters - ---------- - configspace : ConfigurationSpace - The original ConfigurationSpace specified by the user. - uniform_configspace : ConfigurationSpace - A version of the user-defined ConfigurationSpace where all parameters are uniform (or have their weights removed - in the case of a categorical hyperparameter). - acquisition_function : AbstractAcquisitionFunction | None, defaults to None - challengers : int, defaults to 5000 - Number of challengers. - max_steps: int, defaults to None - [LocalSearch] Maximum number of steps that the local search will perform. - n_steps_plateau_walk: int, defaults to 10 - [LocalSearch] number of steps during a plateau walk before local search terminates. - local_search_iterations: int, defaults to 10 - [Local Search] number of local search iterations. - prior_sampling_fraction: float, defaults to 0.5 - The ratio of random samples that are taken from the user-defined ConfigurationSpace, as opposed to the uniform - version. - seed : int, defaults to 0 - """ - - def __init__( - self, - configspace: ConfigurationSpace, - uniform_configspace: ConfigurationSpace, - acquisition_function: AbstractAcquisitionFunction | None = None, - challengers: int = 5000, - max_steps: int | None = None, - n_steps_plateau_walk: int = 10, - local_search_iterations: int = 10, - prior_sampling_fraction: float = 0.5, - seed: int = 0, - ) -> None: - super().__init__( - acquisition_function, - configspace, - challengers=challengers, - seed=seed, - ) - - self._prior_random_search = RandomSearch( - acquisition_function=acquisition_function, - configspace=configspace, - seed=seed, - ) - - self._uniform_random_search = RandomSearch( - acquisition_function=acquisition_function, - configspace=uniform_configspace, - seed=seed, - ) - - self._local_search = LocalSearch( - acquisition_function=acquisition_function, - configspace=configspace, - max_steps=max_steps, - n_steps_plateau_walk=n_steps_plateau_walk, - seed=seed, - ) - - self._local_search_iterations = local_search_iterations - self._prior_sampling_fraction = prior_sampling_fraction - - def _maximize( - self, - previous_configs: list[Configuration], - n_points: int, - ) -> list[tuple[float, Configuration]]: - - # Get configurations sorted by EI - next_configs_by_prior_random_search_sorted = self._prior_random_search._maximize( - previous_configs, - round(n_points * self._prior_sampling_fraction), - _sorted=True, - ) - - # Get configurations sorted by EI - next_configs_by_uniform_random_search_sorted = self._uniform_random_search._maximize( - previous_configs, - round(n_points * (1 - self._prior_sampling_fraction)), - _sorted=True, - ) - next_configs_by_random_search_sorted = [] - next_configs_by_random_search_sorted.extend(next_configs_by_prior_random_search_sorted) - next_configs_by_random_search_sorted.extend(next_configs_by_uniform_random_search_sorted) - - next_configs_by_local_search = self._local_search._maximize( - previous_configs, - self._local_search_iterations, - additional_start_points=next_configs_by_random_search_sorted, - ) - - # Having the configurations from random search, sorted by their - # acquisition function value is important for the first few iterations - # of SMAC. As long as the random forest predicts constant value, we - # want to use only random configurations. Having them at the begging of - # the list ensures this (even after adding the configurations by local - # search, and then sorting them) - next_configs_by_acq_value = next_configs_by_random_search_sorted + next_configs_by_local_search - next_configs_by_acq_value.sort(reverse=True, key=lambda x: x[0]) - logger.debug( - "First 5 acq func (origin) values of selected configurations: %s", - str([[_[0], _[1].origin] for _ in next_configs_by_acq_value[:5]]), - ) - - return next_configs_by_acq_value diff --git a/smac/acquisition/maximizer/local_search.py b/smac/acquisition/maximizer/local_search.py index c6f545f9a..3ef1ae96e 100644 --- a/smac/acquisition/maximizer/local_search.py +++ b/smac/acquisition/maximizer/local_search.py @@ -90,9 +90,9 @@ def _maximize( n_points: int, additional_start_points: list[tuple[float, Configuration]] | None = None, ) -> list[tuple[float, Configuration]]: - """Start a local search from the given startpoints. Iteratively collect neighbours + """Start a local search from the given start points. Iteratively collect neighbours using Configspace.utils.get_one_exchange_neighbourhood and evaluate them. - If the new config is better than the current best, the local search is coninued from the + If the new config is better than the current best, the local search is continued from the new config. Quit if either the max number of steps is reached or @@ -149,16 +149,22 @@ def _get_initial_points( list[Configuration] A list of initial points/configurations. """ - if len(previous_configs) == 0: - init_points = self._configspace.sample_configuration(size=n_points) - else: + sampled_points = [] + init_points = [] + n_init_points = n_points + if len(previous_configs) < n_points: + sampled_points = self._configspace.sample_configuration(size=n_points - len(previous_configs)) + n_init_points = len(previous_configs) + if not isinstance(sampled_points, list): + sampled_points = [sampled_points] + if len(previous_configs) > 0: init_points = self._get_init_points_from_previous_configs( previous_configs, - n_points, + n_init_points, additional_start_points, ) - return init_points + return sampled_points + init_points def _get_init_points_from_previous_configs( self, @@ -187,7 +193,7 @@ def _get_init_points_from_previous_configs( previous_configs: list[Configuration] Previous configuration (e.g., from the runhistory). n_points: int - Number of initial points to be generated. + Number of initial points to be generated; selected from previous configs (+ random configs if necessary). additional_start_points: list[tuple[float, Configuration]] | None Additional starting points. @@ -198,10 +204,6 @@ def _get_init_points_from_previous_configs( """ assert self._acquisition_function is not None - # configurations with the highest previous EI - configs_previous_runs_sorted = self._sort_by_acquisition_value(previous_configs) - configs_previous_runs_sorted = [conf[1] for conf in configs_previous_runs_sorted[:n_points]] - # configurations with the lowest predictive cost, check for None to make unit tests work if self._acquisition_function.model is not None: conf_array = convert_configurations_to_array(previous_configs) @@ -228,14 +230,13 @@ def _get_init_points_from_previous_configs( previous_configs_sorted_by_cost = [] if additional_start_points is not None: - additional_start_points = [asp[1] for asp in additional_start_points[:n_points]] + additional_start_points = [asp[1] for asp in additional_start_points] else: additional_start_points = [] init_points = [] init_points_as_set: set[Configuration] = set() for cand in itertools.chain( - configs_previous_runs_sorted, previous_configs_sorted_by_cost, additional_start_points, ): diff --git a/smac/facade/abstract_facade.py b/smac/facade/abstract_facade.py index a5db5ab52..6ca49b057 100644 --- a/smac/facade/abstract_facade.py +++ b/smac/facade/abstract_facade.py @@ -117,12 +117,15 @@ def __init__( runhistory_encoder: AbstractRunHistoryEncoder | None = None, config_selector: ConfigSelector | None = None, logging_level: int | Path | Literal[False] | None = None, - callbacks: list[Callback] = [], + callbacks: list[Callback] = None, overwrite: bool = False, dask_client: Client | None = None, ): setup_logging(logging_level) + if callbacks is None: + callbacks = [] + if model is None: model = self.get_model(scenario) @@ -190,7 +193,7 @@ def __init__( # In case of multiple jobs, we need to wrap the runner again using DaskParallelRunner if (n_workers := scenario.n_workers) > 1 or dask_client is not None: - if dask_client is not None: + if dask_client is not None and n_workers > 1: logger.warning( "Provided `dask_client`. Ignore `scenario.n_workers`, directly set `n_workers` in `dask_client`." ) diff --git a/smac/facade/algorithm_configuration_facade.py b/smac/facade/algorithm_configuration_facade.py index a82e2f92c..58229f4b7 100644 --- a/smac/facade/algorithm_configuration_facade.py +++ b/smac/facade/algorithm_configuration_facade.py @@ -125,7 +125,7 @@ def get_intensifier( def get_initial_design( # type: ignore scenario: Scenario, *, - additional_configs: list[Configuration] = [], + additional_configs: list[Configuration] = None, ) -> DefaultInitialDesign: """Returns an initial design, which returns the default configuration. @@ -134,6 +134,8 @@ def get_initial_design( # type: ignore additional_configs: list[Configuration], defaults to [] Adds additional configurations to the initial design. """ + if additional_configs is None: + additional_configs = [] return DefaultInitialDesign( scenario=scenario, additional_configs=additional_configs, diff --git a/smac/facade/blackbox_facade.py b/smac/facade/blackbox_facade.py index 9e2d4b2d1..a5e1a18ab 100644 --- a/smac/facade/blackbox_facade.py +++ b/smac/facade/blackbox_facade.py @@ -11,6 +11,7 @@ from smac.facade.abstract_facade import AbstractFacade from smac.initial_design.sobol_design import SobolInitialDesign from smac.intensifier.intensifier import Intensifier +from smac.main.config_selector import ConfigSelector from smac.model.gaussian_process.abstract_gaussian_process import ( AbstractGaussianProcess, ) @@ -240,7 +241,7 @@ def get_initial_design( # type: ignore n_configs: int | None = None, n_configs_per_hyperparamter: int = 8, max_ratio: float = 0.25, - additional_configs: list[Configuration] = [], + additional_configs: list[Configuration] = None, ) -> SobolInitialDesign: """Returns a Sobol design instance. @@ -259,6 +260,8 @@ def get_initial_design( # type: ignore additional_configs: list[Configuration], defaults to [] Adds additional configurations to the initial design. """ + if additional_configs is None: + additional_configs = [] return SobolInitialDesign( scenario=scenario, n_configs=n_configs, @@ -309,3 +312,15 @@ def get_runhistory_encoder( ) -> RunHistoryEncoder: """Returns the default runhistory encoder.""" return RunHistoryEncoder(scenario) + + @staticmethod + def get_config_selector( + scenario: Scenario, + *, + retrain_after: int = 1, + retries: int = 16, + ) -> ConfigSelector: + """Returns the default configuration selector.""" + return super(BlackBoxFacade, BlackBoxFacade).get_config_selector( + scenario, retrain_after=retrain_after, retries=retries + ) diff --git a/smac/facade/hyperparameter_optimization_facade.py b/smac/facade/hyperparameter_optimization_facade.py index 70a85517c..3ed09cfde 100644 --- a/smac/facade/hyperparameter_optimization_facade.py +++ b/smac/facade/hyperparameter_optimization_facade.py @@ -138,7 +138,7 @@ def get_initial_design( # type: ignore n_configs: int | None = None, n_configs_per_hyperparamter: int = 10, max_ratio: float = 0.25, - additional_configs: list[Configuration] = [], + additional_configs: list[Configuration] | None = None, ) -> SobolInitialDesign: """Returns a Sobol design instance. @@ -178,7 +178,7 @@ def get_random_design( # type: ignore probability : float, defaults to 0.2 Probability that a configuration will be drawn at random. """ - return ProbabilityRandomDesign(probability=probability) + return ProbabilityRandomDesign(probability=probability, seed=scenario.seed) @staticmethod def get_multi_objective_algorithm( # type: ignore diff --git a/smac/facade/multi_fidelity_facade.py b/smac/facade/multi_fidelity_facade.py index fdd52d8a4..717162c09 100644 --- a/smac/facade/multi_fidelity_facade.py +++ b/smac/facade/multi_fidelity_facade.py @@ -67,7 +67,7 @@ def get_initial_design( # type: ignore n_configs: int | None = None, n_configs_per_hyperparamter: int = 10, max_ratio: float = 0.25, - additional_configs: list[Configuration] = [], + additional_configs: list[Configuration] = None, ) -> RandomInitialDesign: """Returns a random initial design. @@ -80,12 +80,14 @@ def get_initial_design( # type: ignore Number of initial configurations per hyperparameter. For example, if my configuration space covers five hyperparameters and ``n_configs_per_hyperparameter`` is set to 10, then 50 initial configurations will be samples. - max_ratio: float, defaults to 0.1 + max_ratio: float, defaults to 0.25 Use at most ``scenario.n_trials`` * ``max_ratio`` number of configurations in the initial design. Additional configurations are not affected by this parameter. additional_configs: list[Configuration], defaults to [] Adds additional configurations to the initial design. """ + if additional_configs is None: + additional_configs = [] return RandomInitialDesign( scenario=scenario, n_configs=n_configs, diff --git a/smac/facade/random_facade.py b/smac/facade/random_facade.py index d5e269534..78038ff2e 100644 --- a/smac/facade/random_facade.py +++ b/smac/facade/random_facade.py @@ -92,7 +92,7 @@ def get_intensifier( def get_initial_design( scenario: Scenario, *, - additional_configs: list[Configuration] = [], + additional_configs: list[Configuration] = None, ) -> DefaultInitialDesign: """Returns an initial design, which returns the default configuration. @@ -101,6 +101,8 @@ def get_initial_design( additional_configs: list[Configuration], defaults to [] Adds additional configurations to the initial design. """ + if additional_configs is None: + additional_configs = [] return DefaultInitialDesign( scenario=scenario, additional_configs=additional_configs, diff --git a/smac/intensifier/abstract_intensifier.py b/smac/intensifier/abstract_intensifier.py index b94486727..cb537e9cc 100644 --- a/smac/intensifier/abstract_intensifier.py +++ b/smac/intensifier/abstract_intensifier.py @@ -571,8 +571,12 @@ def update_incumbents(self, config: Configuration) -> None: if len(previous_incumbents) == len(new_incumbents): if previous_incumbents == new_incumbents: - # No changes in the incumbents - self._remove_rejected_config(config_id) + # No changes in the incumbents, we need this clause because we can't use set difference then + if config_id in new_incumbent_ids: + self._remove_rejected_config(config_id) + else: + # config worse than incumbents and thus rejected + self._add_rejected_config(config_id) return else: # In this case, we have to determine which config replaced which incumbent and reject it diff --git a/smac/main/config_selector.py b/smac/main/config_selector.py index 7b2053eee..4e3574d58 100644 --- a/smac/main/config_selector.py +++ b/smac/main/config_selector.py @@ -91,7 +91,7 @@ def _set_components( acquisition_maximizer: AbstractAcquisitionMaximizer, acquisition_function: AbstractAcquisitionFunction, random_design: AbstractRandomDesign, - callbacks: list[Callback] = [], + callbacks: list[Callback] = None, ) -> None: self._runhistory = runhistory self._runhistory_encoder = runhistory_encoder @@ -99,7 +99,7 @@ def _set_components( self._acquisition_maximizer = acquisition_maximizer self._acquisition_function = acquisition_function self._random_design = random_design - self._callbacks = callbacks + self._callbacks = callbacks if callbacks is not None else [] self._initial_design_configs = initial_design.select_configurations() if len(self._initial_design_configs) == 0: @@ -213,7 +213,6 @@ def __iter__(self) -> Iterator[Configuration]: # Now we maximize the acquisition function challengers = self._acquisition_maximizer.maximize( previous_configs, - n_points=self._retrain_after, random_design=self._random_design, ) @@ -282,7 +281,7 @@ def _collect_data(self) -> tuple[np.ndarray, np.ndarray, np.ndarray]: if X.shape[0] >= self._min_trials: self._considered_budgets = [b] - # TODO: Add running configs + # Possible add running configs? configs_array = self._runhistory_encoder.get_configurations(budget_subset=self._considered_budgets) return X, Y, configs_array diff --git a/smac/model/random_forest/abstract_random_forest.py b/smac/model/random_forest/abstract_random_forest.py index d4f1f7ce3..e407331be 100644 --- a/smac/model/random_forest/abstract_random_forest.py +++ b/smac/model/random_forest/abstract_random_forest.py @@ -6,6 +6,7 @@ from ConfigSpace import ( CategoricalHyperparameter, Constant, + OrdinalHyperparameter, UniformFloatHyperparameter, UniformIntegerHyperparameter, ) @@ -36,12 +37,14 @@ def _impute_inactive(self, X: np.ndarray) -> np.ndarray: self._conditional[idx] = True if isinstance(hp, CategoricalHyperparameter): self._impute_values[idx] = len(hp.choices) + elif isinstance(hp, OrdinalHyperparameter): + self._impute_values[idx] = len(hp.sequence) elif isinstance(hp, (UniformFloatHyperparameter, UniformIntegerHyperparameter)): self._impute_values[idx] = -1 elif isinstance(hp, Constant): self._impute_values[idx] = 1 else: - raise ValueError + raise ValueError(f"Unsupported hyperparameter type: {type(hp)}") if self._conditional[idx] is True: nonfinite_mask = ~np.isfinite(X[:, idx]) diff --git a/smac/model/random_forest/random_forest.py b/smac/model/random_forest/random_forest.py index 6d7840d2a..634e0af06 100644 --- a/smac/model/random_forest/random_forest.py +++ b/smac/model/random_forest/random_forest.py @@ -87,7 +87,9 @@ def __init__( self._rf_opts.compute_law_of_total_variance = False self._rf: BinaryForest | None = None self._log_y = log_y - self._rng = regression.default_random_engine(seed) + + # Case to `int` incase we get an `np.integer` type + self._rng = regression.default_random_engine(int(seed)) self._n_trees = n_trees self._n_points_per_tree = n_points_per_tree diff --git a/smac/runhistory/encoder/abstract_encoder.py b/smac/runhistory/encoder/abstract_encoder.py index 42454771e..9a891229a 100644 --- a/smac/runhistory/encoder/abstract_encoder.py +++ b/smac/runhistory/encoder/abstract_encoder.py @@ -42,17 +42,17 @@ class AbstractRunHistoryEncoder: def __init__( self, scenario: Scenario, - considered_states: list[StatusType] = [ - StatusType.SUCCESS, - StatusType.CRASHED, - StatusType.MEMORYOUT, - ], - lower_budget_states: list[StatusType] = [], + considered_states: list[StatusType] = None, + lower_budget_states: list[StatusType] = None, scale_percentage: int = 5, seed: int | None = None, ) -> None: if considered_states is None: - raise TypeError("No success states are given.") + considered_states = [ + StatusType.SUCCESS, + StatusType.CRASHED, + StatusType.MEMORYOUT, + ] if seed is None: seed = scenario.seed @@ -62,7 +62,7 @@ def __init__( self._scale_percentage = scale_percentage self._n_objectives = scenario.count_objectives() self._algorithm_walltime_limit = scenario.trial_walltime_limit - self._lower_budget_states = lower_budget_states + self._lower_budget_states = lower_budget_states if lower_budget_states is not None else [] self._considered_states = considered_states self._instances = scenario.instances diff --git a/smac/runhistory/runhistory.py b/smac/runhistory/runhistory.py index aaf88889c..c71384107 100644 --- a/smac/runhistory/runhistory.py +++ b/smac/runhistory/runhistory.py @@ -178,7 +178,7 @@ def add( budget: float | None = None, starttime: float = 0.0, endtime: float = 0.0, - additional_info: dict[str, Any] = {}, + additional_info: dict[str, Any] = None, force_update: bool = False, ) -> None: """Adds a new trial to the RunHistory. @@ -205,6 +205,8 @@ def add( raise TypeError("Configuration must not be None.") elif not isinstance(config, Configuration): raise TypeError("Configuration is not of type Configuration, but %s." % type(config)) + if additional_info is None: + additional_info = {} # Squeeze is important to reduce arrays with one element # to scalars. diff --git a/smac/runner/abstract_runner.py b/smac/runner/abstract_runner.py index 2631bd966..4e15cf4b5 100644 --- a/smac/runner/abstract_runner.py +++ b/smac/runner/abstract_runner.py @@ -50,8 +50,10 @@ class AbstractRunner(ABC): def __init__( self, scenario: Scenario, - required_arguments: list[str] = [], + required_arguments: list[str] = None, ): + if required_arguments is None: + required_arguments = [] self._scenario = scenario self._required_arguments = required_arguments diff --git a/smac/runner/dask_runner.py b/smac/runner/dask_runner.py index b9aade401..d4bb528bd 100644 --- a/smac/runner/dask_runner.py +++ b/smac/runner/dask_runner.py @@ -91,7 +91,7 @@ def __init__( ) if self._scenario.output_directory is not None: - self._scheduler_file = self._scenario.output_directory / ".dask_scheduler_file" + self._scheduler_file = Path(self._scenario.output_directory, ".dask_scheduler_file") self._client.write_scheduler_file(scheduler_file=str(self._scheduler_file)) else: # We just use their set up diff --git a/smac/runner/target_function_runner.py b/smac/runner/target_function_runner.py index 5431f5ed4..8a14c2fc9 100644 --- a/smac/runner/target_function_runner.py +++ b/smac/runner/target_function_runner.py @@ -45,8 +45,10 @@ def __init__( self, scenario: Scenario, target_function: Callable, - required_arguments: list[str] = [], + required_arguments: list[str] = None, ): + if required_arguments is None: + required_arguments = [] super().__init__(scenario=scenario, required_arguments=required_arguments) self._target_function = target_function diff --git a/smac/runner/target_function_script_runner.py b/smac/runner/target_function_script_runner.py index 17feffc98..5ab558906 100644 --- a/smac/runner/target_function_script_runner.py +++ b/smac/runner/target_function_script_runner.py @@ -51,8 +51,10 @@ def __init__( self, target_function: str, scenario: Scenario, - required_arguments: list[str] = [], + required_arguments: list[str] = None, ): + if required_arguments is None: + required_arguments = [] super().__init__(scenario=scenario, required_arguments=required_arguments) self._target_function = target_function diff --git a/tests/test_acquisition/test_maximizers.py b/tests/test_acquisition/test_maximizers.py index 4cd4110f3..6d373c237 100644 --- a/tests/test_acquisition/test_maximizers.py +++ b/tests/test_acquisition/test_maximizers.py @@ -30,7 +30,6 @@ from smac.acquisition.function import EI from smac.acquisition.maximizer import ( DifferentialEvolution, - LocalAndSortedPriorRandomSearch, LocalAndSortedRandomSearch, LocalSearch, RandomSearch, @@ -278,7 +277,7 @@ def __call__(self, X): random_configs = configspace.sample_configuration(size=100) points = ls._get_initial_points(random_configs, n_points=5, additional_start_points=None) - assert len(points) == 10 + assert len(points) == 5 # -------------------------------------------------------------- @@ -342,7 +341,6 @@ def test_local_and_random_search(configspace, acquisition_function): assert v_old >= v v_old = v - assert "Acquisition Function Maximizer: Random Search (sorted)" in config_origins assert "Acquisition Function Maximizer: Local Search" in config_origins @@ -390,26 +388,26 @@ def __call__(self, arrays): budget_kwargs = {"max_steps": 2, "n_steps_plateau_walk": 2, "local_search_iterations": 2} - prs_0 = LocalAndSortedPriorRandomSearch( - configspace_prior, - configspace_rosenbrock, - AcquisitionFunction(), + prs_0 = LocalAndSortedRandomSearch( + configspace=configspace_prior, + uniform_configspace=configspace_rosenbrock, + acquisition_function=AcquisitionFunction(), prior_sampling_fraction=0, **budget_kwargs, ) - prs_05 = LocalAndSortedPriorRandomSearch( - configspace_prior, - configspace_rosenbrock, - AcquisitionFunction(), + prs_05 = LocalAndSortedRandomSearch( + configspace=configspace_prior, + uniform_configspace=configspace_rosenbrock, + acquisition_function=AcquisitionFunction(), prior_sampling_fraction=0.9, **budget_kwargs, ) - prs_1 = LocalAndSortedPriorRandomSearch( - configspace_prior, - configspace_rosenbrock, - AcquisitionFunction(), + prs_1 = LocalAndSortedRandomSearch( + configspace=configspace_prior, + uniform_configspace=configspace_rosenbrock, + acquisition_function=AcquisitionFunction(), prior_sampling_fraction=1, **budget_kwargs, ) diff --git a/tests/test_intensifier/test_abstract_intensifier.py b/tests/test_intensifier/test_abstract_intensifier.py index ce980c49b..b8dc91a1c 100644 --- a/tests/test_intensifier/test_abstract_intensifier.py +++ b/tests/test_intensifier/test_abstract_intensifier.py @@ -109,6 +109,48 @@ def test_incumbent_selection_multi_objective(make_scenario, configspace_small, m assert intensifier.get_incumbents() == [config] +def test_config_rejection_single_objective(configspace_small, make_scenario): + """ Tests whether configs are rejected properly if they are worse than the incumbent. """ + scenario = make_scenario(configspace_small, use_instances=False) + runhistory = RunHistory() + intensifier = Intensifier(scenario=scenario) + intensifier.runhistory = runhistory + + configs = configspace_small.sample_configuration(3) + + runhistory.add(config=configs[0], + cost=5, + time=0.0, + seed=0, + status=StatusType.SUCCESS, + force_update=True) + intensifier.update_incumbents(configs[0]) + + assert intensifier._rejected_config_ids == [] + + # add config that yielded better results, updating incumbent and sending prior incumbent to rejected + runhistory.add(config=configs[1], + cost=1, + time=0.0, + seed=0, + status=StatusType.SUCCESS, + force_update=True) + intensifier.update_incumbents(config=configs[1]) + + assert intensifier._rejected_config_ids == [1] + + # add config that is no better should thus go to rejected + runhistory.add(config=configs[2], + cost=1, + time=0.0, + seed=0, + status=StatusType.SUCCESS, + force_update=True) + intensifier.update_incumbents(config=configs[2]) + + assert intensifier._rejected_config_ids == [1, 3] + + def test_incumbent_differences(make_scenario, configspace_small): pass diff --git a/tests/test_model/test_rf.py b/tests/test_model/test_rf.py index a30995b23..e59b2ded1 100644 --- a/tests/test_model/test_rf.py +++ b/tests/test_model/test_rf.py @@ -257,24 +257,36 @@ def test_with_ordinal(): def test_impute_inactive_hyperparameters(): cs = ConfigurationSpace(seed=0) - a = cs.add_hyperparameter(CategoricalHyperparameter("a", [0, 1])) + a = cs.add_hyperparameter(CategoricalHyperparameter("a", [0, 1, 2])) b = cs.add_hyperparameter(CategoricalHyperparameter("b", [0, 1])) c = cs.add_hyperparameter(UniformFloatHyperparameter("c", 0, 1)) + d = cs.add_hyperparameter(OrdinalHyperparameter("d", [0, 1, 2])) cs.add_condition(EqualsCondition(b, a, 1)) cs.add_condition(EqualsCondition(c, a, 0)) + cs.add_condition(EqualsCondition(d, a, 2)) configs = cs.sample_configuration(size=100) config_array = convert_configurations_to_array(configs) for line in config_array: if line[0] == 0: assert np.isnan(line[1]) + assert np.isnan(line[3]) elif line[0] == 1: assert np.isnan(line[2]) + assert np.isnan(line[3]) + elif line[0] == 2: + assert np.isnan(line[1]) + assert np.isnan(line[2]) model = RandomForest(configspace=cs) config_array = model._impute_inactive(config_array) for line in config_array: if line[0] == 0: assert line[1] == 2 + assert line[3] == 3 elif line[0] == 1: assert line[2] == -1 + assert line[3] == 3 + elif line[0] == 2: + assert line[1] == 2 + assert line[2] == -1 diff --git a/tests/test_runhistory/test_runhistory_encoder.py b/tests/test_runhistory/test_runhistory_encoder.py index 7b30bb76a..e3a8f5730 100644 --- a/tests/test_runhistory/test_runhistory_encoder.py +++ b/tests/test_runhistory/test_runhistory_encoder.py @@ -13,6 +13,9 @@ from smac.runhistory.encoder.encoder import RunHistoryEncoder from smac.runner.abstract_runner import StatusType +from ConfigSpace import Configuration +from ConfigSpace.hyperparameters import CategoricalHyperparameter + @pytest.fixture def configs(configspace_small): @@ -39,70 +42,127 @@ def test_transform(runhistory, make_scenario, configspace_small, configs): ) # Normal encoder - encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory + + # TODO: Please replace with the more general solution once ConfigSpace 1.0 + # upper = np.array([hp.upper_vectorized for hp in space.values()]) + # lower = np.array([hp.lower_vectorized for hp in space.values()]) + # - + # Categoricals are upperbounded by their size, rest of hyperparameters are + # upperbounded by 1. + upper_bounds = { + hp.name: (hp.get_size() - 1) + if isinstance(hp, CategoricalHyperparameter) + else 1.0 + for hp in configspace_small.get_hyperparameters() + } + # Need to ensure they match the order in the Configuration vectorized form + sorted_by_indices = sorted( + upper_bounds.items(), + key=lambda x: configspace_small._hyperparameter_idx[x[0]], + ) + upper = np.array([upper_bound for _, upper_bound in sorted_by_indices]) + lower = 0.0 + X1, Y1 = encoder.transform() assert Y1.tolist() == [[1.0], [5.0]] - assert ((X1 <= 1.0) & (X1 >= 0.0)).all() + assert ((X1 <= upper) & (X1 >= lower)).all() # Log encoder - encoder = RunHistoryLogEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryLogEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryLogScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryLogScaledEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryScaledEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryInverseScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryInverseScaledEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistorySqrtScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistorySqrtScaledEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryEIPSEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryEIPSEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() -def test_transform_conditionals(runhistory, make_scenario, configspace_large, configs): - configs = configspace_large.sample_configuration(20) +def test_transform_conditionals(runhistory, make_scenario, configspace_large): scenario = make_scenario(configspace_large) + config_1 = Configuration( + configspace_large, + values={ + "activation": "tanh", + "n_layer": 5, + "n_neurons": 27, + "solver": "lbfgs", + }, + ) + config_2 = Configuration( + configspace_large, + values={ + "activation": "tanh", + "batch_size": 47, + "learning_rate": "adaptive", + "learning_rate_init": 0.6673206111956781, + "n_layer": 3, + "n_neurons": 88, + "solver": "sgd", + }, + ) runhistory.add( - config=configs[0], + config=config_1, cost=1, time=1, status=StatusType.SUCCESS, ) runhistory.add( - config=configs[2], + config=config_2, cost=5, time=4, status=StatusType.SUCCESS, ) - encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform() @@ -124,7 +184,9 @@ def test_multi_objective(runhistory, make_scenario, configspace_small, configs): # Multi objective algorithm must be set with pytest.raises(AssertionError): - encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory _, Y = encoder.transform() @@ -180,7 +242,9 @@ def test_ignore(runhistory, make_scenario, configspace_small, configs): ) # Normal encoder - encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X1, Y1 = encoder.transform() @@ -219,10 +283,14 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs): budget=2, ) - runhistory.add(config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2) + runhistory.add( + config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2 + ) # Normal encoder - encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform(budget_subset=[2]) assert Y.tolist() == [[99999999]] @@ -251,10 +319,14 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs): budget=2, ) - runhistory.add(config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2) + runhistory.add( + config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2 + ) # Normal encoder - encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory X, Y = encoder.transform(budget_subset=[2]) assert Y.tolist() == [[99999999]] @@ -266,12 +338,20 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs): def test_lower_budget_states(runhistory, make_scenario, configspace_small, configs): """Tests lower budgets based on budget subset and considered states.""" scenario = make_scenario(configspace_small) - encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) + encoder = RunHistoryEncoder( + scenario=scenario, considered_states=[StatusType.SUCCESS] + ) encoder.runhistory = runhistory - runhistory.add(config=configs[0], cost=1, time=1, status=StatusType.SUCCESS, budget=3) - runhistory.add(config=configs[0], cost=2, time=2, status=StatusType.SUCCESS, budget=4) - runhistory.add(config=configs[0], cost=3, time=4, status=StatusType.TIMEOUT, budget=5) + runhistory.add( + config=configs[0], cost=1, time=1, status=StatusType.SUCCESS, budget=3 + ) + runhistory.add( + config=configs[0], cost=2, time=2, status=StatusType.SUCCESS, budget=4 + ) + runhistory.add( + config=configs[0], cost=3, time=4, status=StatusType.TIMEOUT, budget=5 + ) # We request a higher budget but can't find it, so we expect an empty list X, Y = encoder.transform(budget_subset=[500]) From 8ece46d7d97a16f1b8a0c68addb2aa520afb2836 Mon Sep 17 00:00:00 2001 From: Helena Graf Date: Thu, 16 May 2024 15:37:28 +0200 Subject: [PATCH 35/57] Add version 2.1.0 to docs drop-down --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 198361fb9..afb4d2605 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ "version": version, "versions": { f"v{version}": "#", + "v2.1.0": "https://automl.github.io/SMAC3/v2.2.0/", "v2.0.1": "https://automl.github.io/SMAC3/v2.0.1/", "v2.0.0": "https://automl.github.io/SMAC3/v2.0.0/", "v2.0.0b1": "https://automl.github.io/SMAC3/v2.0.0b1/", From a58d29fb9df67367f06c94de9949517c9d9aaf16 Mon Sep 17 00:00:00 2001 From: Marius Lindauer Date: Fri, 14 Jun 2024 15:50:19 +0200 Subject: [PATCH 36/57] Create FUNDING.yml (#1113) --- .github/FUNDING.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..978ac4577 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +github: automl From 2674173c174483a1c5612408d552faf30391fa4e Mon Sep 17 00:00:00 2001 From: "C. Benjamins" <75323339+benjamc@users.noreply.github.com> Date: Tue, 16 Jul 2024 17:31:09 +0200 Subject: [PATCH 37/57] Fix dependencies (numpy + ConfigSpace) and docs (#1122) * fix(dependency): np.NaN -> np.nan New(er) numpy does not support np.NaN anymore * Update CHANGELOG.md * fix(build): restrict package versions Numpy and ConfigSpace's API have changed. Do not incorporate changes yet. * Update CHANGELOG.md * Fix broken link in docs --- CHANGELOG.md | 6 ++++++ docs/3_getting_started.rst | 2 +- setup.py | 4 ++-- smac/model/random_forest/random_forest.py | 2 +- smac/runhistory/encoder/abstract_encoder.py | 6 +++--- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6768b689..97e18d559 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# 2.1.1 + +## Dependencies +- Update numpy NaN (#1122) and restrict numpy and ConfigSpace versions + + # 2.1.0 ## Improvements diff --git a/docs/3_getting_started.rst b/docs/3_getting_started.rst index dc568184a..dbc287368 100644 --- a/docs/3_getting_started.rst +++ b/docs/3_getting_started.rst @@ -27,7 +27,7 @@ ranges and default values. "species": ["mouse", "cat", "dog"], # Categorical }) -Please see the documentation of `ConfigSpace `_ for more details. +Please see the documentation of `ConfigSpace `_ for more details. Target Function diff --git a/setup.py b/setup.py index 27ec7e695..9e8467e4b 100644 --- a/setup.py +++ b/setup.py @@ -57,11 +57,11 @@ def read_file(filepath: str) -> str: include_package_data=True, python_requires=">=3.8", install_requires=[ - "numpy>=1.23.3", + "numpy>=1.23.3,<2.0.0", "scipy>=1.9.2", "psutil", "pynisher>=1.0.0", - "ConfigSpace>=0.6.1", + "ConfigSpace>=0.6.1,<1.0.0", "joblib", "scikit-learn>=1.1.2", "pyrfr>=0.9.0", diff --git a/smac/model/random_forest/random_forest.py b/smac/model/random_forest/random_forest.py index 634e0af06..72685803f 100644 --- a/smac/model/random_forest/random_forest.py +++ b/smac/model/random_forest/random_forest.py @@ -213,7 +213,7 @@ def _predict( third_dimension = max(max_num_leaf_data, third_dimension) # Transform list of 2d arrays into a 3d array - preds_as_array = np.zeros((X.shape[0], self._rf_opts.num_trees, third_dimension)) * np.NaN + preds_as_array = np.zeros((X.shape[0], self._rf_opts.num_trees, third_dimension)) * np.nan for i, preds_per_tree in enumerate(all_preds): for j, pred in enumerate(preds_per_tree): preds_as_array[i, j, : len(pred)] = pred diff --git a/smac/runhistory/encoder/abstract_encoder.py b/smac/runhistory/encoder/abstract_encoder.py index 9a891229a..e9de8b14c 100644 --- a/smac/runhistory/encoder/abstract_encoder.py +++ b/smac/runhistory/encoder/abstract_encoder.py @@ -80,9 +80,9 @@ def __init__( ) # Learned statistics - self._min_y = np.array([np.NaN] * self._n_objectives) - self._max_y = np.array([np.NaN] * self._n_objectives) - self._percentile = np.array([np.NaN] * self._n_objectives) + self._min_y = np.array([np.nan] * self._n_objectives) + self._max_y = np.array([np.nan] * self._n_objectives) + self._percentile = np.array([np.nan] * self._n_objectives) self._multi_objective_algorithm: AbstractMultiObjectiveAlgorithm | None = None self._runhistory: RunHistory | None = None From c7a48f02699c124843ae1b2bc81dff6e0691826a Mon Sep 17 00:00:00 2001 From: Eisuke Kawashima Date: Wed, 17 Jul 2024 00:39:22 +0900 Subject: [PATCH 38/57] chore: fix URLs (#1114) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Change stale-reminder to recent-reminder (#1095) * Version 2.1.0 (#1108) * Hacking in the sum cost for MO * Fix mypi * Bugfix * Make mypi happy * Fix tests * Make format * Remove num_obj in MO optimizer * A lot of restructuring * Solve importings * Solve importings * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Updated changelog * Still fixing :) * fix docstrings * Change directory of tests * Fix ambiguous variable * Del newlines * fix mypy * mypy and doc fix * Removed int type checking * fix tests * Always return List of incs * fix path issue in example * maint boing facade * Better docstring * Return values if no bounds passed * Removed import * Return only one incumbent * Rename n_optimizers -> n_workers * Update example * Update docstring * Remove legacy function * Update to new signature * Update changelog.md * Format black * Update test * Please pre-commit * Merge development * Fix tests * Fix typos * Format black * Updated text * Increased version * Improved readme * SMAC logo 50% * SMAC logo 50% * Removed dev-2.0 * Added doc workflow for latest tag * Fixing docs latest tag * Fixing docs latest tag * First steps new intensifier * Make it work again * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * Test issues * FIX formatting * Link to old documentation * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * FIX formatting * Fix unit tests * Add annotation * corrected log prob for tophat prior * Removed configurations from callback * Fixed tests * Fix mypy * Save docs as latest version too * Enabled all tests again * Make version an env * Added last version to docs * Global variable fix * Added v before version number * New Intensifier Implementation (#888) - Completely reimplemented the intensifiers (including Successive Halving and Hyperband): All intensifiers support multi-fidelity, multi-objective and multi-threading by nature now. - Expected behaviour for ask-and-tell interface ensured (also for Successive Halving). - Continuing a run is now fully supported. - Added more examples. - Updated documentation based on new implementation. * First benchmark steps * Added a warning * Set xi to 0.01 * Incorporated walltime in trajectory * Working version of benchmark * Changed requirements * Changed xi back to 0.0 again * No duplicate budgets * Fixed bug which updated tracker wrongly * Bugfix: Delete correct bracket+stage * Fixed tests * Added costs to the trajectory * Changed initial design max ratio to 0.25 * Make format * Added more details to intensifier * Finished benchmark * Added benchmark source files * Fixed tests because of initial design changes * Improved runhistory API * Fixed typo (#894) * Merge doc fixes * Doc fixes (#893) Co-authored-by: timruhkopf Co-authored-by: Carolin Benjamins Co-authored-by: dengdifan Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs * Make format * Satisfied mypy * Enabled more tests * Increased version * Mypy * Mypy * Mypy * Fixed critical bug when using highest budget for inc selection * Added SH+HB comparison * Stuff * Updated report * Fixed docstring * Fixed image path * Removed vscode folder * Bugfixes * Doc fixes * Updated changelog * Make isort * Fix typos * Removed new line * Added alpha2 to docs selection * update description * Fix 531 correctly create integers in initial design This PR fixes a bug in which integer hyperparameters would not be created correctly in the initial design. This went unnoticed, except in the case where it was part of a condition and the function deactivate_inactive_hyperparameters was called. This PR fixes this bug by mapping the random float that was sampled by, for example and LHD or Sobol sequence, to the float representation of an integer by calling _transform and _inverse_transform of the integer hyperparameter. * Add contact paragraph to README. * Increase days until stale issues are closed by stalebot (#904) * Edit changelog * Add issue number * Closes #907 * Clarify origin of configurations (#908) * Random forest speed up (#903) Co-authored-by: René Sass * Add: workflow to update pre-commit versions (#874) Co-authored-by: René Sass * Add Git-Flow Workflow Specification * [DOC] acquisition function maximizers * Add stale reminder email action * Increase version number of ConfigSpace * Add guideline for updating copyright notice years to CONTRIBUTING.md * Update requirement for pyrfr (#920) * Update pre-commit: isort,flake8 version numbers * Format black * Fix mypy * Ac bench (#929) * Add Branin AC Task * Add ac optimization type * Nicer plotting * Add ac mode * Update ac task * Make new smac env name adjustable * Increase python version for smac 1.4 * Update benchmark * Update plotting * Goodby comments * Fix path * Fix 1.4 wrapper * Update benchmark results --------- Co-authored-by: Carolin Benjamins * [DOC] added documentation on the incumbent update * Feature/add version to make publish (#931) * Add version, correct package name and use bash in makefile * Increase version * Increase version in makefile, add places with versions to make publish * Fix typo * Transform integer feature names to strings (#930) * Transform int features to str * Fix pre-commit * Update version to 2.0.0 * Update version to 2.0.0 * Update CHANGELOG.md * Add information on deepcave to the readme * Correct doc version * fix(runhistory): set id of the sampled config (#951) * fix: Set config_id of config in runhistory * ci: Set dependabot to target development branch (#963) * feat: Option preventing SMAC setting up logging (#947) * Version 2.0.0 (#898) * Update runhistory2epm.py * pre_commit * Updated changelog * Fixed comment * Updated changelog * Some small improvements * Removed trailing space * Updated changelog * Restricted pynisher * Updated changelog * Link JMLR paper * Update bibtex entry Before: arxiv paper, now journal article. * Fixed citation in docs * Fixed callback typos * BOinG for mixed hyperparameter search space * fix format * maint * maint documentation * maint pre-commit * fix workflow * maint * Add dependabot for workflow files * Bump actions/setup-python from 2 to 4 (#849) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/checkout from 2 to 3 (#850) Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * maint * Updated stale bot * Updated stale options again * Fixed typing * Increased version * rename check_point_in_ss to check_subspace_point * Make format * Uncomment test procedure * docstrings for boing facade * avoid typing namespace everywhere in epm_configuration_chooser * fix docstring in rh2epm_boing * Limit on dask * Trying to fix tests * Replaced typing in all files * docstring for boing subspace * Changed feature requests to feature * allow acq_optimizer_local_kwargs to be passed to boing subspace without acq_optimizer_local * fix pre-commit * Added any * Update facade * Fix typing * Match arguments from pSMAC with SMAC4AC signature * Add description of pSMAC to docstring * Add option to pass facade class to pSMAC * Add example for pSMAC * Removed docstring in readme's example * Corrected text * move construct_gp_kernels to smac.epm_gpytorch.boing_kernels * maint * fix wrongly imported func name in tests * Add reference for pSMAC #609 * Rename example file * Added roadmap to readme * Prettification * Add result merger * Reformat with black * Delete unnecessary import * Fix typing * Inject first trajectory entry from file from first rundir * Add test for ResultMerger * Fix docstring * Get runhistory and trajectory from pSMAC run Via the result merger. Searchers rundirs and merges found runhistories and calculates a new trajectory. * Get trajectory * Add regex to requirements * Fixed MO cache problem * Fix MO tests * Fixing MO in AbstractRacer * Accepting ints * Hacking in the sum cost for MO * Fix mypi * Bugfix * Make mypi happy * Fix tests * Make format * Remove num_obj in MO optimizer * A lot of restructuring * Solve importings * Solve importings * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Updated changelog * Still fixing :) * fix docstrings * Change directory of tests * Fix ambiguous variable * Del newlines * fix mypy * mypy and doc fix * Removed int type checking * fix tests * Always return List of incs * fix path issue in example * maint boing facade * Better docstring * Return values if no bounds passed * Removed import * Return only one incumbent * Rename n_optimizers -> n_workers * Update example * Update docstring * Remove legacy function * Update to new signature * Update changelog.md * Format black * Update test * Please pre-commit * Merge development * Fix tests * Fix typos * Format black * Updated text * Increased version * Improved readme * SMAC logo 50% * SMAC logo 50% * Removed dev-2.0 * Added doc workflow for latest tag * Fixing docs latest tag * Fixing docs latest tag * First steps new intensifier * Make it work again * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * Test issues * FIX formatting * Link to old documentation * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * FIX formatting * Fix unit tests * Add annotation * corrected log prob for tophat prior * Removed configurations from callback * Fixed tests * Fix mypy * Save docs as latest version too * Enabled all tests again * Make version an env * Added last version to docs * Global variable fix * Added v before version number * New Intensifier Implementation (#888) - Completely reimplemented the intensifiers (including Successive Halving and Hyperband): All intensifiers support multi-fidelity, multi-objective and multi-threading by nature now. - Expected behaviour for ask-and-tell interface ensured (also for Successive Halving). - Continuing a run is now fully supported. - Added more examples. - Updated documentation based on new implementation. * First benchmark steps * Added a warning * Set xi to 0.01 * Incorporated walltime in trajectory * Working version of benchmark * Changed requirements * Changed xi back to 0.0 again * No duplicate budgets * Fixed bug which updated tracker wrongly * Bugfix: Delete correct bracket+stage * Fixed tests * Added costs to the trajectory * Changed initial design max ratio to 0.25 * Make format * Added more details to intensifier * Finished benchmark * Added benchmark source files * Fixed tests because of initial design changes * Improved runhistory API * Fixed typo (#894) * Merge doc fixes * Doc fixes (#893) Co-authored-by: timruhkopf Co-authored-by: Carolin Benjamins Co-authored-by: dengdifan Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs * Make format * Satisfied mypy * Enabled more tests * Increased version * Mypy * Mypy * Mypy * Fixed critical bug when using highest budget for inc selection * Added SH+HB comparison * Stuff * Updated report * Fixed docstring * Fixed image path * Removed vscode folder * Bugfixes * Doc fixes * Updated changelog * Make isort * Fix typos * Removed new line * Added alpha2 to docs selection * update description * Fix 531 correctly create integers in initial design This PR fixes a bug in which integer hyperparameters would not be created correctly in the initial design. This went unnoticed, except in the case where it was part of a condition and the function deactivate_inactive_hyperparameters was called. This PR fixes this bug by mapping the random float that was sampled by, for example and LHD or Sobol sequence, to the float representation of an integer by calling _transform and _inverse_transform of the integer hyperparameter. * Add contact paragraph to README. * Increase days until stale issues are closed by stalebot (#904) * Edit changelog * Add issue number * Closes #907 * Clarify origin of configurations (#908) * Random forest speed up (#903) Co-authored-by: René Sass * Add: workflow to update pre-commit versions (#874) Co-authored-by: René Sass * Add Git-Flow Workflow Specification * [DOC] acquisition function maximizers * Add stale reminder email action * Increase version number of ConfigSpace * Add guideline for updating copyright notice years to CONTRIBUTING.md * Update requirement for pyrfr (#920) * Update pre-commit: isort,flake8 version numbers * Format black * Fix mypy * Ac bench (#929) * Add Branin AC Task * Add ac optimization type * Nicer plotting * Add ac mode * Update ac task * Make new smac env name adjustable * Increase python version for smac 1.4 * Update benchmark * Update plotting * Goodby comments * Fix path * Fix 1.4 wrapper * Update benchmark results --------- Co-authored-by: Carolin Benjamins * [DOC] added documentation on the incumbent update * Feature/add version to make publish (#931) * Add version, correct package name and use bash in makefile * Increase version * Increase version in makefile, add places with versions to make publish * Fix typo * Transform integer feature names to strings (#930) * Transform int features to str * Fix pre-commit * Update version to 2.0.0 * Update version to 2.0.0 * Update CHANGELOG.md * Add information on deepcave to the readme * Correct doc version --------- Co-authored-by: dengdifan <33290713+dengdifan@users.noreply.github.com> Co-authored-by: dengdifan Co-authored-by: Matthias Feurer Co-authored-by: Carolin Benjamins Co-authored-by: eddiebergman Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Tim Ruhkopf * feat: Option preventing SMAC setting up logging * fix: Run isort * Bump peter-evans/create-pull-request from 3 to 5 (#958) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 3 to 5. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v3...v5) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/setup-python from 2 to 4 (#945) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat: Option preventing SMAC setting up logging * fix: Run isort * fix: Add to changelog * feat: Option preventing SMAC setting up logging * fix: Run isort --------- Signed-off-by: dependabot[bot] Co-authored-by: René Sass Co-authored-by: dengdifan <33290713+dengdifan@users.noreply.github.com> Co-authored-by: dengdifan Co-authored-by: Matthias Feurer Co-authored-by: Carolin Benjamins Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Tim Ruhkopf * Citation update (#961) * update CITATION.cff * update order in CITATION.cff * fix citation date * fix citation * maint * Adjust hyperband configuration distribution across brackets * Compute the actual differences between the isb keys. (#957) * Compute the actual differences between the isb keys. * Added change to CHANGELOG.md * Adjust schedule for stale reminder * fix(logging): Prevent automatic logging setup at init (#970) * fix(logging): Prevent automatic logging setup at init * fix(logging): Rely on `setup_logging` to handle arg * Fix validate method of smbo, update docstring. * Allow callbacks to be added to a specific index, make callback registering public * Update CHANGELOG.md * Fix broken references (#988) * Adapt developer install instructions to include pre-commit installation (#994) * [Feature] Initial Design now supports executing the default config provided by the user by means of the configspace he generated. (and is no longer ignored) * Pipe dask_client through facade for exec on hpc (#983) * Pipe dask_client through facade for exec on hpc * Handle pynisher's memory arg Before: Could only pass int via SMAC with the default unit B, but now tuple/list is possible * Update CHANGELOG.md * Make format * No printi --------- Co-authored-by: Carolin Benjamins * [Bug-fix] Pr was failing due to mutable additional configs default * [doc-fix] the count of initial design configs is computed slightly differently * Feature/metadata callback (#999) * Add example for using a callback to log run metadata to a file * Add issue number in changelog * Changelog formatting * Simplify metadata example * Simplify metadata example * Put callback in smac directory to allow importing it in custom repos * Move callback files to callback directory * Edit changelog * Correct metadata callback argument types * Additional metadata arguments in example * Set metadata default values in callback * Edit comment to fix PR * Documentation/dask client example (#1001) * Update requirements (dask_jobqueue) * Fix broken link * Ignore parallel example * Add parallelization example * Remove imports, add comment abt dask client * Add E's information * Update docs * Add comment about n_workers --------- Co-authored-by: Carolin Benjamins * Documentation/dask client example (#1001) * Update requirements (dask_jobqueue) * Fix broken link * Ignore parallel example * Add parallelization example * Remove imports, add comment abt dask client * Add E's information * Update docs * Add comment about n_workers --------- Co-authored-by: Carolin Benjamins * Feature/dask client (#1002) * Add info message that scenario.n_workers is ignored When passing a dask client, directly set n_workers in client. * Change info to warning --------- Co-authored-by: Carolin Benjamins * Update smac version (#1003) * Update smac version * Add hint on github release in make publish * Update copyright year * check if config in rh when storing state (#997) * check if config in rh when storing state * maint * remove unnecessary changes * Adapt developer install instructions to include pre-commit installation (#994) * Pipe dask_client through facade for exec on hpc (#983) * Pipe dask_client through facade for exec on hpc * Handle pynisher's memory arg Before: Could only pass int via SMAC with the default unit B, but now tuple/list is possible * Update CHANGELOG.md * Make format * No printi --------- Co-authored-by: Carolin Benjamins * update changelog * Increase tolerance for runhistory restoration test --------- Co-authored-by: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: Helena Graf * Add a workaround to be able to pass a dataset via dask.scatter (#993) * [feat] Support dask.scatter * [fix] Add shared_data arg in abstract_facade * Address the comments by Eddie * [fix] Fix errors raised by Eddie * [format] Apply black locally * [fix] Get retries back to 16 as it was a mistake * [doc] Update CHANGELOG --------- Co-authored-by: helegraf Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> * Adding Binder links for example (#976) * binder * maint * fix dependencies * maint docs * Revert "maint docs" This reverts commit 4fa1a0fae8dfa0b646e43273d50f72ddd76bf191. * add requirement --------- Co-authored-by: Difan Deng * Remove commit message from dependabot PRs (#1009) * Bugfix/fix broken link (#1012) * Remove commit message from dependabot PRs (#1010) * rename check_point_in_ss to check_subspace_point * Make format * Uncomment test procedure * docstrings for boing facade * avoid typing namespace everywhere in epm_configuration_chooser * fix docstring in rh2epm_boing * Limit on dask * Trying to fix tests * Replaced typing in all files * docstring for boing subspace * Changed feature requests to feature * allow acq_optimizer_local_kwargs to be passed to boing subspace without acq_optimizer_local * fix pre-commit * Added any * Update facade * Fix typing * Match arguments from pSMAC with SMAC4AC signature * Add description of pSMAC to docstring * Add option to pass facade class to pSMAC * Add example for pSMAC * Removed docstring in readme's example * Corrected text * move construct_gp_kernels to smac.epm_gpytorch.boing_kernels * maint * fix wrongly imported func name in tests * Add reference for pSMAC #609 * Rename example file * Added roadmap to readme * Prettification * Add result merger * Reformat with black * Delete unnecessary import * Fix typing * Inject first trajectory entry from file from first rundir * Add test for ResultMerger * Fix docstring * Get runhistory and trajectory from pSMAC run Via the result merger. Searchers rundirs and merges found runhistories and calculates a new trajectory. * Get trajectory * Add regex to requirements * Fixed MO cache problem * Fix MO tests * Fixing MO in AbstractRacer * Accepting ints * Hacking in the sum cost for MO * Fix mypi * Bugfix * Make mypi happy * Fix tests * Make format * Remove num_obj in MO optimizer * A lot of restructuring * Solve importings * Solve importings * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Updated changelog * Still fixing :) * fix docstrings * Change directory of tests * Fix ambiguous variable * Del newlines * fix mypy * mypy and doc fix * Removed int type checking * fix tests * Always return List of incs * fix path issue in example * maint boing facade * Better docstring * Return values if no bounds passed * Removed import * Return only one incumbent * Rename n_optimizers -> n_workers * Update example * Update docstring * Remove legacy function * Update to new signature * Update changelog.md * Format black * Update test * Please pre-commit * Merge development * Fix tests * Fix typos * Format black * Updated text * Increased version * Improved readme * SMAC logo 50% * SMAC logo 50% * Removed dev-2.0 * Added doc workflow for latest tag * Fixing docs latest tag * Fixing docs latest tag * First steps new intensifier * Make it work again * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * Test issues * FIX formatting * Link to old documentation * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * FIX formatting * Fix unit tests * Add annotation * corrected log prob for tophat prior * Removed configurations from callback * Fixed tests * Fix mypy * Save docs as latest version too * Enabled all tests again * Make version an env * Added last version to docs * Global variable fix * Added v before version number * New Intensifier Implementation (#888) - Completely reimplemented the intensifiers (including Successive Halving and Hyperband): All intensifiers support multi-fidelity, multi-objective and multi-threading by nature now. - Expected behaviour for ask-and-tell interface ensured (also for Successive Halving). - Continuing a run is now fully supported. - Added more examples. - Updated documentation based on new implementation. * First benchmark steps * Added a warning * Set xi to 0.01 * Incorporated walltime in trajectory * Working version of benchmark * Changed requirements * Changed xi back to 0.0 again * No duplicate budgets * Fixed bug which updated tracker wrongly * Bugfix: Delete correct bracket+stage * Fixed tests * Added costs to the trajectory * Changed initial design max ratio to 0.25 * Make format * Added more details to intensifier * Finished benchmark * Added benchmark source files * Fixed tests because of initial design changes * Improved runhistory API * Fixed typo (#894) * Merge doc fixes * Doc fixes (#893) Co-authored-by: timruhkopf Co-authored-by: Carolin Benjamins Co-authored-by: dengdifan Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs * Make format * Satisfied mypy * Enabled more tests * Increased version * Mypy * Mypy * Mypy * Fixed critical bug when using highest budget for inc selection * Added SH+HB comparison * Stuff * Updated report * Fixed docstring * Fixed image path * Removed vscode folder * Bugfixes * Doc fixes * Updated changelog * Make isort * Fix typos * Removed new line * Added alpha2 to docs selection * update description * Fix 531 correctly create integers in initial design This PR fixes a bug in which integer hyperparameters would not be created correctly in the initial design. This went unnoticed, except in the case where it was part of a condition and the function deactivate_inactive_hyperparameters was called. This PR fixes this bug by mapping the random float that was sampled by, for example and LHD or Sobol sequence, to the float representation of an integer by calling _transform and _inverse_transform of the integer hyperparameter. * Add contact paragraph to README. * Increase days until stale issues are closed by stalebot (#904) * Edit changelog * Add issue number * Closes #907 * Clarify origin of configurations (#908) * Random forest speed up (#903) Co-authored-by: René Sass * Add: workflow to update pre-commit versions (#874) Co-authored-by: René Sass * Add Git-Flow Workflow Specification * [DOC] acquisition function maximizers * Add stale reminder email action * Increase version number of ConfigSpace * Add guideline for updating copyright notice years to CONTRIBUTING.md * Update requirement for pyrfr (#920) * Update pre-commit: isort,flake8 version numbers * Format black * Fix mypy * Ac bench (#929) * Add Branin AC Task * Add ac optimization type * Nicer plotting * Add ac mode * Update ac task * Make new smac env name adjustable * Increase python version for smac 1.4 * Update benchmark * Update plotting * Goodby comments * Fix path * Fix 1.4 wrapper * Update benchmark results --------- Co-authored-by: Carolin Benjamins * [DOC] added documentation on the incumbent update * Feature/add version to make publish (#931) * Add version, correct package name and use bash in makefile * Increase version * Increase version in makefile, add places with versions to make publish * Fix typo * Transform integer feature names to strings (#930) * Transform int features to str * Fix pre-commit * Update version to 2.0.0 * Update version to 2.0.0 * Update CHANGELOG.md * Add information on deepcave to the readme * Correct doc version * fix(runhistory): set id of the sampled config (#951) * fix: Set config_id of config in runhistory * ci: Set dependabot to target development branch (#963) * feat: Option preventing SMAC setting up logging (#947) * Version 2.0.0 (#898) * Update runhistory2epm.py * pre_commit * Updated changelog * Fixed comment * Updated changelog * Some small improvements * Removed trailing space * Updated changelog * Restricted pynisher * Updated changelog * Link JMLR paper * Update bibtex entry Before: arxiv paper, now journal article. * Fixed citation in docs * Fixed callback typos * BOinG for mixed hyperparameter search space * fix format * maint * maint documentation * maint pre-commit * fix workflow * maint * Add dependabot for workflow files * Bump actions/setup-python from 2 to 4 (#849) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/checkout from 2 to 3 (#850) Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * maint * Updated stale bot * Updated stale options again * Fixed typing * Increased version * rename check_point_in_ss to check_subspace_point * Make format * Uncomment test procedure * docstrings for boing facade * avoid typing namespace everywhere in epm_configuration_chooser * fix docstring in rh2epm_boing * Limit on dask * Trying to fix tests * Replaced typing in all files * docstring for boing subspace * Changed feature requests to feature * allow acq_optimizer_local_kwargs to be passed to boing subspace without acq_optimizer_local * fix pre-commit * Added any * Update facade * Fix typing * Match arguments from pSMAC with SMAC4AC signature * Add description of pSMAC to docstring * Add option to pass facade class to pSMAC * Add example for pSMAC * Removed docstring in readme's example * Corrected text * move construct_gp_kernels to smac.epm_gpytorch.boing_kernels * maint * fix wrongly imported func name in tests * Add reference for pSMAC #609 * Rename example file * Added roadmap to readme * Prettification * Add result merger * Reformat with black * Delete unnecessary import * Fix typing * Inject first trajectory entry from file from first rundir * Add test for ResultMerger * Fix docstring * Get runhistory and trajectory from pSMAC run Via the result merger. Searchers rundirs and merges found runhistories and calculates a new trajectory. * Get trajectory * Add regex to requirements * Fixed MO cache problem * Fix MO tests * Fixing MO in AbstractRacer * Accepting ints * Hacking in the sum cost for MO * Fix mypi * Bugfix * Make mypi happy * Fix tests * Make format * Remove num_obj in MO optimizer * A lot of restructuring * Solve importings * Solve importings * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Updated changelog * Still fixing :) * fix docstrings * Change directory of tests * Fix ambiguous variable * Del newlines * fix mypy * mypy and doc fix * Removed int type checking * fix tests * Always return List of incs * fix path issue in example * maint boing facade * Better docstring * Return values if no bounds passed * Removed import * Return only one incumbent * Rename n_optimizers -> n_workers * Update example * Update docstring * Remove legacy function * Update to new signature * Update changelog.md * Format black * Update test * Please pre-commit * Merge development * Fix tests * Fix typos * Format black * Updated text * Increased version * Improved readme * SMAC logo 50% * SMAC logo 50% * Removed dev-2.0 * Added doc workflow for latest tag * Fixing docs latest tag * Fixing docs latest tag * First steps new intensifier * Make it work again * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * Test issues * FIX formatting * Link to old documentation * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * FIX formatting * Fix unit tests * Add annotation * corrected log prob for tophat prior * Removed configurations from callback * Fixed tests * Fix mypy * Save docs as latest version too * Enabled all tests again * Make version an env * Added last version to docs * Global variable fix * Added v before version number * New Intensifier Implementation (#888) - Completely reimplemented the intensifiers (including Successive Halving and Hyperband): All intensifiers support multi-fidelity, multi-objective and multi-threading by nature now. - Expected behaviour for ask-and-tell interface ensured (also for Successive Halving). - Continuing a run is now fully supported. - Added more examples. - Updated documentation based on new implementation. * First benchmark steps * Added a warning * Set xi to 0.01 * Incorporated walltime in trajectory * Working version of benchmark * Changed requirements * Changed xi back to 0.0 again * No duplicate budgets * Fixed bug which updated tracker wrongly * Bugfix: Delete correct bracket+stage * Fixed tests * Added costs to the trajectory * Changed initial design max ratio to 0.25 * Make format * Added more details to intensifier * Finished benchmark * Added benchmark source files * Fixed tests because of initial design changes * Improved runhistory API * Fixed typo (#894) * Merge doc fixes * Doc fixes (#893) Co-authored-by: timruhkopf Co-authored-by: Carolin Benjamins Co-authored-by: dengdifan Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs * Make format * Satisfied mypy * Enabled more tests * Increased version * Mypy * Mypy * Mypy * Fixed critical bug when using highest budget for inc selection * Added SH+HB comparison * Stuff * Updated report * Fixed docstring * Fixed image path * Removed vscode folder * Bugfixes * Doc fixes * Updated changelog * Make isort * Fix typos * Removed new line * Added alpha2 to docs selection * update description * Fix 531 correctly create integers in initial design This PR fixes a bug in which integer hyperparameters would not be created correctly in the initial design. This went unnoticed, except in the case where it was part of a condition and the function deactivate_inactive_hyperparameters was called. This PR fixes this bug by mapping the random float that was sampled by, for example and LHD or Sobol sequence, to the float representation of an integer by calling _transform and _inverse_transform of the integer hyperparameter. * Add contact paragraph to README. * Increase days until stale issues are closed by stalebot (#904) * Edit changelog * Add issue number * Closes #907 * Clarify origin of configurations (#908) * Random forest speed up (#903) Co-authored-by: René Sass * Add: workflow to update pre-commit versions (#874) Co-authored-by: René Sass * Add Git-Flow Workflow Specification * [DOC] acquisition function maximizers * Add stale reminder email action * Increase version number of ConfigSpace * Add guideline for updating copyright notice years to CONTRIBUTING.md * Update requirement for pyrfr (#920) * Update pre-commit: isort,flake8 version numbers * Format black * Fix mypy * Ac bench (#929) * Add Branin AC Task * Add ac optimization type * Nicer plotting * Add ac mode * Update ac task * Make new smac env name adjustable * Increase python version for smac 1.4 * Update benchmark * Update plotting * Goodby comments * Fix path * Fix 1.4 wrapper * Update benchmark results --------- Co-authored-by: Carolin Benjamins * [DOC] added documentation on the incumbent update * Feature/add version to make publish (#931) * Add version, correct package name and use bash in makefile * Increase version * Increase version in makefile, add places with versions to make publish * Fix typo * Transform integer feature names to strings (#930) * Transform int features to str * Fix pre-commit * Update version to 2.0.0 * Update version to 2.0.0 * Update CHANGELOG.md * Add information on deepcave to the readme * Correct doc version --------- Co-authored-by: dengdifan <33290713+dengdifan@users.noreply.github.com> Co-authored-by: dengdifan Co-authored-by: Matthias Feurer Co-authored-by: Carolin Benjamins Co-authored-by: eddiebergman Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Tim Ruhkopf * feat: Option preventing SMAC setting up logging * fix: Run isort * Bump peter-evans/create-pull-request from 3 to 5 (#958) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 3 to 5. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v3...v5) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/setup-python from 2 to 4 (#945) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat: Option preventing SMAC setting up logging * fix: Run isort * fix: Add to changelog * feat: Option preventing SMAC setting up logging * fix: Run isort --------- Signed-off-by: dependabot[bot] Co-authored-by: René Sass Co-authored-by: dengdifan <33290713+dengdifan@users.noreply.github.com> Co-authored-by: dengdifan Co-authored-by: Matthias Feurer Co-authored-by: Carolin Benjamins Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Tim Ruhkopf * Citation update (#961) * update CITATION.cff * update order in CITATION.cff * fix citation date * fix citation * maint * Adjust hyperband configuration distribution across brackets * Compute the actual differences between the isb keys. (#957) * Compute the actual differences between the isb keys. * Added change to CHANGELOG.md * Adjust schedule for stale reminder * fix(logging): Prevent automatic logging setup at init (#970) * fix(logging): Prevent automatic logging setup at init * fix(logging): Rely on `setup_logging` to handle arg * Fix validate method of smbo, update docstring. * Allow callbacks to be added to a specific index, make callback registering public * Update CHANGELOG.md * Fix broken references (#988) * Adapt developer install instructions to include pre-commit installation (#994) * [Feature] Initial Design now supports executing the default config provided by the user by means of the configspace he generated. (and is no longer ignored) * Pipe dask_client through facade for exec on hpc (#983) * Pipe dask_client through facade for exec on hpc * Handle pynisher's memory arg Before: Could only pass int via SMAC with the default unit B, but now tuple/list is possible * Update CHANGELOG.md * Make format * No printi --------- Co-authored-by: Carolin Benjamins * [Bug-fix] Pr was failing due to mutable additional configs default * [doc-fix] the count of initial design configs is computed slightly differently * Feature/metadata callback (#999) * Add example for using a callback to log run metadata to a file * Add issue number in changelog * Changelog formatting * Simplify metadata example * Simplify metadata example * Put callback in smac directory to allow importing it in custom repos * Move callback files to callback directory * Edit changelog * Correct metadata callback argument types * Additional metadata arguments in example * Set metadata default values in callback * Edit comment to fix PR * Documentation/dask client example (#1001) * Update requirements (dask_jobqueue) * Fix broken link * Ignore parallel example * Add parallelization example * Remove imports, add comment abt dask client * Add E's information * Update docs * Add comment about n_workers --------- Co-authored-by: Carolin Benjamins * Documentation/dask client example (#1001) * Update requirements (dask_jobqueue) * Fix broken link * Ignore parallel example * Add parallelization example * Remove imports, add comment abt dask client * Add E's information * Update docs * Add comment about n_workers --------- Co-authored-by: Carolin Benjamins * Feature/dask client (#1002) * Add info message that scenario.n_workers is ignored When passing a dask client, directly set n_workers in client. * Change info to warning --------- Co-authored-by: Carolin Benjamins * Update smac version (#1003) * Update smac version * Add hint on github release in make publish * Update copyright year * check if config in rh when storing state (#997) * check if config in rh when storing state * maint * remove unnecessary changes * Adapt developer install instructions to include pre-commit installation (#994) * Pipe dask_client through facade for exec on hpc (#983) * Pipe dask_client through facade for exec on hpc * Handle pynisher's memory arg Before: Could only pass int via SMAC with the default unit B, but now tuple/list is possible * Update CHANGELOG.md * Make format * No printi --------- Co-authored-by: Carolin Benjamins * update changelog * Increase tolerance for runhistory restoration test --------- Co-authored-by: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: Helena Graf * Add a workaround to be able to pass a dataset via dask.scatter (#993) * [feat] Support dask.scatter * [fix] Add shared_data arg in abstract_facade * Address the comments by Eddie * [fix] Fix errors raised by Eddie * [format] Apply black locally * [fix] Get retries back to 16 as it was a mistake * [doc] Update CHANGELOG --------- Co-authored-by: helegraf Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> * Adding Binder links for example (#976) * binder * maint * fix dependencies * maint docs * Revert "maint docs" This reverts commit 4fa1a0fae8dfa0b646e43273d50f72ddd76bf191. * add requirement --------- Co-authored-by: Difan Deng * Remove commit message from dependabot PRs (#1009) --------- Signed-off-by: dependabot[bot] Co-authored-by: dengdifan Co-authored-by: René Sass Co-authored-by: Carolin Benjamins Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Matthias Feurer Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Eddie Bergman Co-authored-by: Tim Ruhkopf Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> Co-authored-by: Aron Bahram Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Co-authored-by: Difan Deng * Fix broken link --------- Signed-off-by: dependabot[bot] Co-authored-by: dengdifan Co-authored-by: René Sass Co-authored-by: Carolin Benjamins Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Matthias Feurer Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Eddie Bergman Co-authored-by: Tim Ruhkopf Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> Co-authored-by: Aron Bahram Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Co-authored-by: Difan Deng * Drop torch requirements (#1015) * Remove torch requirements * Fix format --------- Co-authored-by: Carolin Benjamins * Remove leftover CLI entry points (#1014) * Fix incumbent selection in case of SH+MO * Update CHANGELOG.md * Feature/GitHub actions pr draft (#1008) Update the workflows only to trigger extensive tests for PRs that are not in the draft stage --------- Co-authored-by: timruhkopf Co-authored-by: Helena Graf * Update stale report to include issues with recent interactions (#1025) * Add recently updated issues to list of reminders. * Add recently updated issues to list of reminders. * Add recently updated issues to list of reminders. * Raise error if resource limitation and parallelization is requested (#1023) * Add warnings for parallelism doc * Check for parallelization + pynisher Does not work together, throw error. Does not work bc Dask pickles jobs and pynisher runs the target function in a process. * Fix whitespaces --------- Co-authored-by: Carolin Benjamins * Raise an error for invalid scatter data (#1018) * [enhance] Add an error for scatter data * Update ChangeLog --------- Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> * fix(configspace): shows all keys in print (#1043) * Fix callback order (#1040) Co-authored-by: Carolin Benjamins * fix: dtypes (#1044) * Fix(config_selector): Budget checking (#1039) * Fix: target runner with partial func (#1045) * Feature/improve documentation (#1028) * Experimental instruction for installing SMAC in windows via WSL * More detailed documentation regarding continuing runs * Fixes * Fixes * Fixes * Update docs/10_experimental.rst Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> * Fixes from PR * Fixes from PR * Fixes from PR * Fixes from PR * Fixes from PR --------- Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> * Handle configspace as dictionary in mlp example (#1057) * Handle configspace as dictionary in mlp example * Adapt sgd loss to newest scikit-learn version * Add change to changelog * Handle configspace as dictionary in parego example * Correct get usage * Raise version number * Add missing period in docs. * allow rf to impute OrdinalHyperparameter (#1065) * allow rf to impute OrdinalHyperparameter * Add missing period in docs. * Add issue number in changelog. --------- Co-authored-by: Sarah Krebs * fix dask_scheduler_file path (#1055) * fix dask_scheduler_file path * update change log --------- Co-authored-by: dengdifan * Fix config rejection for #1068 (#1069) * racing: make sure config only deleted from rejected configs if among incumbents upon update * test case for incumbent rejection * update code comments * added missing newline ahead of new test case * update change log --------- Co-authored-by: dengdifan Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> * fix(RandomForest): Always cast `seed` to `int` (#1084) * fix(RandomForest): Always cast `seed` to `int` In cases where seeds were generated from some numpy objects, sometimes you'd get back an `np.integer`, which causes the program to crash when communicating with `pyrfr` through `swig`. It seems that swig doesn't know that it's an _int-like_ and so we explicitly cast it to `int`. * Update CHANGELOG.md --------- Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> * Bump actions/checkout from 3 to 4 (#1072) * Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update CHANGELOG.md --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: benjamc * chore: log warn on condition when custom dask client is provided (#1071) * Bump conda-incubator/setup-miniconda from 2 to 3 (#1087) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 2 to 3. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v2...v3) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Propagate the Scenario random seed to get_random_design (#1066) * Propagate the Scenario's random seed to get_random_design * add changelog line --------- Co-authored-by: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> * [#1056] Add example on intensify for cross-validation. (#1061) * Add example on intensify for cross-validation. * Improve example * Address Difan's comment * Update CHANGELOG.md * Change stale-reminder to recent-reminder (#1096) * Change stale-reminder to recent-reminder (#1095) (#1099) * Dont use mutable lists as default args (#1097) * Don't use mutable default argument. If this isn't respected, then additional_configs can be built up overtime when multiple optimizers are created in a single process. This can occur when the use_default_config option is set True and/or when additional_configs are provided. When different ConfigSpaces are provided this can result in an error due to mismatched defaults, for instance. In other cases it can result in non-deterministic behavior due to the random ordering of the configs that get added (e.g., with different parallel pytest runs). - [ ] Cleanup other sources of this error as well elsewhere in the code. - [ ] Add pylint checks for this issue. * fixup some others mutable defaults * add pylint checks * fix more related pylint issues * add changelog line * Update pre-commit config * Update pre-commit config * Remove TODO formulation as an issue already exists for this --------- Co-authored-by: Brian Kroth * Bump actions/setup-python from 4 to 5 (#1089) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump peter-evans/create-pull-request from 5 to 6 (#1093) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 5 to 6. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v5...v6) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * test: Fix upper bounds for `test_transformer()` and no random in `test_transformer_conditional()` (#1102) * test: Don't rely on randomization for the test * test: Include import * test: Fix boundaries for catgoricals in test * test: Missing import * doc: Make todo not more clear about when to upgrade * Change stale-reminder to recent-reminder (#1095) (#1107) * Fix typo in docstring for `MultiFideltyFacade.get_initial_design` (#1104) * Fix typo in docstring for `MultiFideltyFacade.get_initial_design` The docstring specifies 0.1 is used as a default value for max_ratio, however in code 0.25 is used. * Bug/acquisition maximizer sampling (#1106) * Adapt how n_points works in the config_selector.py Also remove LocalAndSortedPriorRandomSearch by integrating it into LocalAndSortedRandomSearch * Change retrain_after to 1 for BlackBOx Facade * Fix bugs in local_and_random_search.py * Edit CHANGELOG.md * fix acq maximizer docstring * call get config selector from super class * Update random and local search to new functionality * Update CHANGELOG.md * Fix sampling of initial points for local search when less previous configs than n_points * Re-format blackbo… * Create FUNDING.yml (#1113) * chore: fix URLs --------- Signed-off-by: dependabot[bot] Co-authored-by: helegraf Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: René Sass Co-authored-by: dengdifan Co-authored-by: Carolin Benjamins Co-authored-by: Katharina Eggensperger Co-authored-by: Matthias Feurer Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: Jeroen Rook Co-authored-by: Eddie Bergman Co-authored-by: Tim Ruhkopf Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> Co-authored-by: Aron Bahram Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Co-authored-by: Difan Deng Co-authored-by: Bastian Zimmermann <10774221+BastianZim@users.noreply.github.com> Co-authored-by: Weihuang Wen Co-authored-by: PhilippBordne <71140732+PhilippBordne@users.noreply.github.com> Co-authored-by: SimonThormeyer <49559340+SimonThormeyer@users.noreply.github.com> Co-authored-by: Brian Kroth Co-authored-by: Brian Kroth Co-authored-by: fleance Co-authored-by: Marius Lindauer Co-authored-by: Eisuke Kawashima --- .github/FUNDING.yml | 3 +++ CHANGELOG.md | 1 - smac/__init__.py | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..978ac4577 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +github: automl diff --git a/CHANGELOG.md b/CHANGELOG.md index 97e18d559..46b37206e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,6 @@ ## Dependencies - Update numpy NaN (#1122) and restrict numpy and ConfigSpace versions - # 2.1.0 ## Improvements diff --git a/smac/__init__.py b/smac/__init__.py index 3241da062..45d7df2d2 100644 --- a/smac/__init__.py +++ b/smac/__init__.py @@ -12,8 +12,8 @@ description = "SMAC3, a Python implementation of 'Sequential Model-based Algorithm Configuration'." url = "https://www.automl.org/" project_urls = { - "Documentation": "https://https://github.com/automl.github.io/SMAC3/main", - "Source Code": "https://github.com/https://github.com/automl/smac", + "Documentation": "https://automl.github.io/SMAC3/main", + "Source Code": "https://github.com/automl/SMAC3", } copyright = f""" Copyright {datetime.date.today().strftime('%Y')}, Marius Lindauer, Katharina Eggensperger, From 7ce2b24b0c62c56a279d10215e37ff3fe054a322 Mon Sep 17 00:00:00 2001 From: "C. Benjamins" <75323339+benjamc@users.noreply.github.com> Date: Tue, 16 Jul 2024 17:51:13 +0200 Subject: [PATCH 39/57] [MF] Determine number of trials for a total optimization budget (#1121) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Change stale-reminder to recent-reminder (#1095) * Version 2.1.0 (#1108) * Hacking in the sum cost for MO * Fix mypi * Bugfix * Make mypi happy * Fix tests * Make format * Remove num_obj in MO optimizer * A lot of restructuring * Solve importings * Solve importings * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Updated changelog * Still fixing :) * fix docstrings * Change directory of tests * Fix ambiguous variable * Del newlines * fix mypy * mypy and doc fix * Removed int type checking * fix tests * Always return List of incs * fix path issue in example * maint boing facade * Better docstring * Return values if no bounds passed * Removed import * Return only one incumbent * Rename n_optimizers -> n_workers * Update example * Update docstring * Remove legacy function * Update to new signature * Update changelog.md * Format black * Update test * Please pre-commit * Merge development * Fix tests * Fix typos * Format black * Updated text * Increased version * Improved readme * SMAC logo 50% * SMAC logo 50% * Removed dev-2.0 * Added doc workflow for latest tag * Fixing docs latest tag * Fixing docs latest tag * First steps new intensifier * Make it work again * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * Test issues * FIX formatting * Link to old documentation * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * FIX formatting * Fix unit tests * Add annotation * corrected log prob for tophat prior * Removed configurations from callback * Fixed tests * Fix mypy * Save docs as latest version too * Enabled all tests again * Make version an env * Added last version to docs * Global variable fix * Added v before version number * New Intensifier Implementation (#888) - Completely reimplemented the intensifiers (including Successive Halving and Hyperband): All intensifiers support multi-fidelity, multi-objective and multi-threading by nature now. - Expected behaviour for ask-and-tell interface ensured (also for Successive Halving). - Continuing a run is now fully supported. - Added more examples. - Updated documentation based on new implementation. * First benchmark steps * Added a warning * Set xi to 0.01 * Incorporated walltime in trajectory * Working version of benchmark * Changed requirements * Changed xi back to 0.0 again * No duplicate budgets * Fixed bug which updated tracker wrongly * Bugfix: Delete correct bracket+stage * Fixed tests * Added costs to the trajectory * Changed initial design max ratio to 0.25 * Make format * Added more details to intensifier * Finished benchmark * Added benchmark source files * Fixed tests because of initial design changes * Improved runhistory API * Fixed typo (#894) * Merge doc fixes * Doc fixes (#893) Co-authored-by: timruhkopf Co-authored-by: Carolin Benjamins Co-authored-by: dengdifan Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs * Make format * Satisfied mypy * Enabled more tests * Increased version * Mypy * Mypy * Mypy * Fixed critical bug when using highest budget for inc selection * Added SH+HB comparison * Stuff * Updated report * Fixed docstring * Fixed image path * Removed vscode folder * Bugfixes * Doc fixes * Updated changelog * Make isort * Fix typos * Removed new line * Added alpha2 to docs selection * update description * Fix 531 correctly create integers in initial design This PR fixes a bug in which integer hyperparameters would not be created correctly in the initial design. This went unnoticed, except in the case where it was part of a condition and the function deactivate_inactive_hyperparameters was called. This PR fixes this bug by mapping the random float that was sampled by, for example and LHD or Sobol sequence, to the float representation of an integer by calling _transform and _inverse_transform of the integer hyperparameter. * Add contact paragraph to README. * Increase days until stale issues are closed by stalebot (#904) * Edit changelog * Add issue number * Closes #907 * Clarify origin of configurations (#908) * Random forest speed up (#903) Co-authored-by: René Sass * Add: workflow to update pre-commit versions (#874) Co-authored-by: René Sass * Add Git-Flow Workflow Specification * [DOC] acquisition function maximizers * Add stale reminder email action * Increase version number of ConfigSpace * Add guideline for updating copyright notice years to CONTRIBUTING.md * Update requirement for pyrfr (#920) * Update pre-commit: isort,flake8 version numbers * Format black * Fix mypy * Ac bench (#929) * Add Branin AC Task * Add ac optimization type * Nicer plotting * Add ac mode * Update ac task * Make new smac env name adjustable * Increase python version for smac 1.4 * Update benchmark * Update plotting * Goodby comments * Fix path * Fix 1.4 wrapper * Update benchmark results --------- Co-authored-by: Carolin Benjamins * [DOC] added documentation on the incumbent update * Feature/add version to make publish (#931) * Add version, correct package name and use bash in makefile * Increase version * Increase version in makefile, add places with versions to make publish * Fix typo * Transform integer feature names to strings (#930) * Transform int features to str * Fix pre-commit * Update version to 2.0.0 * Update version to 2.0.0 * Update CHANGELOG.md * Add information on deepcave to the readme * Correct doc version * fix(runhistory): set id of the sampled config (#951) * fix: Set config_id of config in runhistory * ci: Set dependabot to target development branch (#963) * feat: Option preventing SMAC setting up logging (#947) * Version 2.0.0 (#898) * Update runhistory2epm.py * pre_commit * Updated changelog * Fixed comment * Updated changelog * Some small improvements * Removed trailing space * Updated changelog * Restricted pynisher * Updated changelog * Link JMLR paper * Update bibtex entry Before: arxiv paper, now journal article. * Fixed citation in docs * Fixed callback typos * BOinG for mixed hyperparameter search space * fix format * maint * maint documentation * maint pre-commit * fix workflow * maint * Add dependabot for workflow files * Bump actions/setup-python from 2 to 4 (#849) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/checkout from 2 to 3 (#850) Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * maint * Updated stale bot * Updated stale options again * Fixed typing * Increased version * rename check_point_in_ss to check_subspace_point * Make format * Uncomment test procedure * docstrings for boing facade * avoid typing namespace everywhere in epm_configuration_chooser * fix docstring in rh2epm_boing * Limit on dask * Trying to fix tests * Replaced typing in all files * docstring for boing subspace * Changed feature requests to feature * allow acq_optimizer_local_kwargs to be passed to boing subspace without acq_optimizer_local * fix pre-commit * Added any * Update facade * Fix typing * Match arguments from pSMAC with SMAC4AC signature * Add description of pSMAC to docstring * Add option to pass facade class to pSMAC * Add example for pSMAC * Removed docstring in readme's example * Corrected text * move construct_gp_kernels to smac.epm_gpytorch.boing_kernels * maint * fix wrongly imported func name in tests * Add reference for pSMAC #609 * Rename example file * Added roadmap to readme * Prettification * Add result merger * Reformat with black * Delete unnecessary import * Fix typing * Inject first trajectory entry from file from first rundir * Add test for ResultMerger * Fix docstring * Get runhistory and trajectory from pSMAC run Via the result merger. Searchers rundirs and merges found runhistories and calculates a new trajectory. * Get trajectory * Add regex to requirements * Fixed MO cache problem * Fix MO tests * Fixing MO in AbstractRacer * Accepting ints * Hacking in the sum cost for MO * Fix mypi * Bugfix * Make mypi happy * Fix tests * Make format * Remove num_obj in MO optimizer * A lot of restructuring * Solve importings * Solve importings * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Updated changelog * Still fixing :) * fix docstrings * Change directory of tests * Fix ambiguous variable * Del newlines * fix mypy * mypy and doc fix * Removed int type checking * fix tests * Always return List of incs * fix path issue in example * maint boing facade * Better docstring * Return values if no bounds passed * Removed import * Return only one incumbent * Rename n_optimizers -> n_workers * Update example * Update docstring * Remove legacy function * Update to new signature * Update changelog.md * Format black * Update test * Please pre-commit * Merge development * Fix tests * Fix typos * Format black * Updated text * Increased version * Improved readme * SMAC logo 50% * SMAC logo 50% * Removed dev-2.0 * Added doc workflow for latest tag * Fixing docs latest tag * Fixing docs latest tag * First steps new intensifier * Make it work again * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * Test issues * FIX formatting * Link to old documentation * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * FIX formatting * Fix unit tests * Add annotation * corrected log prob for tophat prior * Removed configurations from callback * Fixed tests * Fix mypy * Save docs as latest version too * Enabled all tests again * Make version an env * Added last version to docs * Global variable fix * Added v before version number * New Intensifier Implementation (#888) - Completely reimplemented the intensifiers (including Successive Halving and Hyperband): All intensifiers support multi-fidelity, multi-objective and multi-threading by nature now. - Expected behaviour for ask-and-tell interface ensured (also for Successive Halving). - Continuing a run is now fully supported. - Added more examples. - Updated documentation based on new implementation. * First benchmark steps * Added a warning * Set xi to 0.01 * Incorporated walltime in trajectory * Working version of benchmark * Changed requirements * Changed xi back to 0.0 again * No duplicate budgets * Fixed bug which updated tracker wrongly * Bugfix: Delete correct bracket+stage * Fixed tests * Added costs to the trajectory * Changed initial design max ratio to 0.25 * Make format * Added more details to intensifier * Finished benchmark * Added benchmark source files * Fixed tests because of initial design changes * Improved runhistory API * Fixed typo (#894) * Merge doc fixes * Doc fixes (#893) Co-authored-by: timruhkopf Co-authored-by: Carolin Benjamins Co-authored-by: dengdifan Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs * Make format * Satisfied mypy * Enabled more tests * Increased version * Mypy * Mypy * Mypy * Fixed critical bug when using highest budget for inc selection * Added SH+HB comparison * Stuff * Updated report * Fixed docstring * Fixed image path * Removed vscode folder * Bugfixes * Doc fixes * Updated changelog * Make isort * Fix typos * Removed new line * Added alpha2 to docs selection * update description * Fix 531 correctly create integers in initial design This PR fixes a bug in which integer hyperparameters would not be created correctly in the initial design. This went unnoticed, except in the case where it was part of a condition and the function deactivate_inactive_hyperparameters was called. This PR fixes this bug by mapping the random float that was sampled by, for example and LHD or Sobol sequence, to the float representation of an integer by calling _transform and _inverse_transform of the integer hyperparameter. * Add contact paragraph to README. * Increase days until stale issues are closed by stalebot (#904) * Edit changelog * Add issue number * Closes #907 * Clarify origin of configurations (#908) * Random forest speed up (#903) Co-authored-by: René Sass * Add: workflow to update pre-commit versions (#874) Co-authored-by: René Sass * Add Git-Flow Workflow Specification * [DOC] acquisition function maximizers * Add stale reminder email action * Increase version number of ConfigSpace * Add guideline for updating copyright notice years to CONTRIBUTING.md * Update requirement for pyrfr (#920) * Update pre-commit: isort,flake8 version numbers * Format black * Fix mypy * Ac bench (#929) * Add Branin AC Task * Add ac optimization type * Nicer plotting * Add ac mode * Update ac task * Make new smac env name adjustable * Increase python version for smac 1.4 * Update benchmark * Update plotting * Goodby comments * Fix path * Fix 1.4 wrapper * Update benchmark results --------- Co-authored-by: Carolin Benjamins * [DOC] added documentation on the incumbent update * Feature/add version to make publish (#931) * Add version, correct package name and use bash in makefile * Increase version * Increase version in makefile, add places with versions to make publish * Fix typo * Transform integer feature names to strings (#930) * Transform int features to str * Fix pre-commit * Update version to 2.0.0 * Update version to 2.0.0 * Update CHANGELOG.md * Add information on deepcave to the readme * Correct doc version --------- Co-authored-by: dengdifan <33290713+dengdifan@users.noreply.github.com> Co-authored-by: dengdifan Co-authored-by: Matthias Feurer Co-authored-by: Carolin Benjamins Co-authored-by: eddiebergman Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Tim Ruhkopf * feat: Option preventing SMAC setting up logging * fix: Run isort * Bump peter-evans/create-pull-request from 3 to 5 (#958) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 3 to 5. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v3...v5) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/setup-python from 2 to 4 (#945) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat: Option preventing SMAC setting up logging * fix: Run isort * fix: Add to changelog * feat: Option preventing SMAC setting up logging * fix: Run isort --------- Signed-off-by: dependabot[bot] Co-authored-by: René Sass Co-authored-by: dengdifan <33290713+dengdifan@users.noreply.github.com> Co-authored-by: dengdifan Co-authored-by: Matthias Feurer Co-authored-by: Carolin Benjamins Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Tim Ruhkopf * Citation update (#961) * update CITATION.cff * update order in CITATION.cff * fix citation date * fix citation * maint * Adjust hyperband configuration distribution across brackets * Compute the actual differences between the isb keys. (#957) * Compute the actual differences between the isb keys. * Added change to CHANGELOG.md * Adjust schedule for stale reminder * fix(logging): Prevent automatic logging setup at init (#970) * fix(logging): Prevent automatic logging setup at init * fix(logging): Rely on `setup_logging` to handle arg * Fix validate method of smbo, update docstring. * Allow callbacks to be added to a specific index, make callback registering public * Update CHANGELOG.md * Fix broken references (#988) * Adapt developer install instructions to include pre-commit installation (#994) * [Feature] Initial Design now supports executing the default config provided by the user by means of the configspace he generated. (and is no longer ignored) * Pipe dask_client through facade for exec on hpc (#983) * Pipe dask_client through facade for exec on hpc * Handle pynisher's memory arg Before: Could only pass int via SMAC with the default unit B, but now tuple/list is possible * Update CHANGELOG.md * Make format * No printi --------- Co-authored-by: Carolin Benjamins * [Bug-fix] Pr was failing due to mutable additional configs default * [doc-fix] the count of initial design configs is computed slightly differently * Feature/metadata callback (#999) * Add example for using a callback to log run metadata to a file * Add issue number in changelog * Changelog formatting * Simplify metadata example * Simplify metadata example * Put callback in smac directory to allow importing it in custom repos * Move callback files to callback directory * Edit changelog * Correct metadata callback argument types * Additional metadata arguments in example * Set metadata default values in callback * Edit comment to fix PR * Documentation/dask client example (#1001) * Update requirements (dask_jobqueue) * Fix broken link * Ignore parallel example * Add parallelization example * Remove imports, add comment abt dask client * Add E's information * Update docs * Add comment about n_workers --------- Co-authored-by: Carolin Benjamins * Documentation/dask client example (#1001) * Update requirements (dask_jobqueue) * Fix broken link * Ignore parallel example * Add parallelization example * Remove imports, add comment abt dask client * Add E's information * Update docs * Add comment about n_workers --------- Co-authored-by: Carolin Benjamins * Feature/dask client (#1002) * Add info message that scenario.n_workers is ignored When passing a dask client, directly set n_workers in client. * Change info to warning --------- Co-authored-by: Carolin Benjamins * Update smac version (#1003) * Update smac version * Add hint on github release in make publish * Update copyright year * check if config in rh when storing state (#997) * check if config in rh when storing state * maint * remove unnecessary changes * Adapt developer install instructions to include pre-commit installation (#994) * Pipe dask_client through facade for exec on hpc (#983) * Pipe dask_client through facade for exec on hpc * Handle pynisher's memory arg Before: Could only pass int via SMAC with the default unit B, but now tuple/list is possible * Update CHANGELOG.md * Make format * No printi --------- Co-authored-by: Carolin Benjamins * update changelog * Increase tolerance for runhistory restoration test --------- Co-authored-by: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: Helena Graf * Add a workaround to be able to pass a dataset via dask.scatter (#993) * [feat] Support dask.scatter * [fix] Add shared_data arg in abstract_facade * Address the comments by Eddie * [fix] Fix errors raised by Eddie * [format] Apply black locally * [fix] Get retries back to 16 as it was a mistake * [doc] Update CHANGELOG --------- Co-authored-by: helegraf Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> * Adding Binder links for example (#976) * binder * maint * fix dependencies * maint docs * Revert "maint docs" This reverts commit 4fa1a0fae8dfa0b646e43273d50f72ddd76bf191. * add requirement --------- Co-authored-by: Difan Deng * Remove commit message from dependabot PRs (#1009) * Bugfix/fix broken link (#1012) * Remove commit message from dependabot PRs (#1010) * rename check_point_in_ss to check_subspace_point * Make format * Uncomment test procedure * docstrings for boing facade * avoid typing namespace everywhere in epm_configuration_chooser * fix docstring in rh2epm_boing * Limit on dask * Trying to fix tests * Replaced typing in all files * docstring for boing subspace * Changed feature requests to feature * allow acq_optimizer_local_kwargs to be passed to boing subspace without acq_optimizer_local * fix pre-commit * Added any * Update facade * Fix typing * Match arguments from pSMAC with SMAC4AC signature * Add description of pSMAC to docstring * Add option to pass facade class to pSMAC * Add example for pSMAC * Removed docstring in readme's example * Corrected text * move construct_gp_kernels to smac.epm_gpytorch.boing_kernels * maint * fix wrongly imported func name in tests * Add reference for pSMAC #609 * Rename example file * Added roadmap to readme * Prettification * Add result merger * Reformat with black * Delete unnecessary import * Fix typing * Inject first trajectory entry from file from first rundir * Add test for ResultMerger * Fix docstring * Get runhistory and trajectory from pSMAC run Via the result merger. Searchers rundirs and merges found runhistories and calculates a new trajectory. * Get trajectory * Add regex to requirements * Fixed MO cache problem * Fix MO tests * Fixing MO in AbstractRacer * Accepting ints * Hacking in the sum cost for MO * Fix mypi * Bugfix * Make mypi happy * Fix tests * Make format * Remove num_obj in MO optimizer * A lot of restructuring * Solve importings * Solve importings * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Updated changelog * Still fixing :) * fix docstrings * Change directory of tests * Fix ambiguous variable * Del newlines * fix mypy * mypy and doc fix * Removed int type checking * fix tests * Always return List of incs * fix path issue in example * maint boing facade * Better docstring * Return values if no bounds passed * Removed import * Return only one incumbent * Rename n_optimizers -> n_workers * Update example * Update docstring * Remove legacy function * Update to new signature * Update changelog.md * Format black * Update test * Please pre-commit * Merge development * Fix tests * Fix typos * Format black * Updated text * Increased version * Improved readme * SMAC logo 50% * SMAC logo 50% * Removed dev-2.0 * Added doc workflow for latest tag * Fixing docs latest tag * Fixing docs latest tag * First steps new intensifier * Make it work again * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * Test issues * FIX formatting * Link to old documentation * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * FIX formatting * Fix unit tests * Add annotation * corrected log prob for tophat prior * Removed configurations from callback * Fixed tests * Fix mypy * Save docs as latest version too * Enabled all tests again * Make version an env * Added last version to docs * Global variable fix * Added v before version number * New Intensifier Implementation (#888) - Completely reimplemented the intensifiers (including Successive Halving and Hyperband): All intensifiers support multi-fidelity, multi-objective and multi-threading by nature now. - Expected behaviour for ask-and-tell interface ensured (also for Successive Halving). - Continuing a run is now fully supported. - Added more examples. - Updated documentation based on new implementation. * First benchmark steps * Added a warning * Set xi to 0.01 * Incorporated walltime in trajectory * Working version of benchmark * Changed requirements * Changed xi back to 0.0 again * No duplicate budgets * Fixed bug which updated tracker wrongly * Bugfix: Delete correct bracket+stage * Fixed tests * Added costs to the trajectory * Changed initial design max ratio to 0.25 * Make format * Added more details to intensifier * Finished benchmark * Added benchmark source files * Fixed tests because of initial design changes * Improved runhistory API * Fixed typo (#894) * Merge doc fixes * Doc fixes (#893) Co-authored-by: timruhkopf Co-authored-by: Carolin Benjamins Co-authored-by: dengdifan Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs * Make format * Satisfied mypy * Enabled more tests * Increased version * Mypy * Mypy * Mypy * Fixed critical bug when using highest budget for inc selection * Added SH+HB comparison * Stuff * Updated report * Fixed docstring * Fixed image path * Removed vscode folder * Bugfixes * Doc fixes * Updated changelog * Make isort * Fix typos * Removed new line * Added alpha2 to docs selection * update description * Fix 531 correctly create integers in initial design This PR fixes a bug in which integer hyperparameters would not be created correctly in the initial design. This went unnoticed, except in the case where it was part of a condition and the function deactivate_inactive_hyperparameters was called. This PR fixes this bug by mapping the random float that was sampled by, for example and LHD or Sobol sequence, to the float representation of an integer by calling _transform and _inverse_transform of the integer hyperparameter. * Add contact paragraph to README. * Increase days until stale issues are closed by stalebot (#904) * Edit changelog * Add issue number * Closes #907 * Clarify origin of configurations (#908) * Random forest speed up (#903) Co-authored-by: René Sass * Add: workflow to update pre-commit versions (#874) Co-authored-by: René Sass * Add Git-Flow Workflow Specification * [DOC] acquisition function maximizers * Add stale reminder email action * Increase version number of ConfigSpace * Add guideline for updating copyright notice years to CONTRIBUTING.md * Update requirement for pyrfr (#920) * Update pre-commit: isort,flake8 version numbers * Format black * Fix mypy * Ac bench (#929) * Add Branin AC Task * Add ac optimization type * Nicer plotting * Add ac mode * Update ac task * Make new smac env name adjustable * Increase python version for smac 1.4 * Update benchmark * Update plotting * Goodby comments * Fix path * Fix 1.4 wrapper * Update benchmark results --------- Co-authored-by: Carolin Benjamins * [DOC] added documentation on the incumbent update * Feature/add version to make publish (#931) * Add version, correct package name and use bash in makefile * Increase version * Increase version in makefile, add places with versions to make publish * Fix typo * Transform integer feature names to strings (#930) * Transform int features to str * Fix pre-commit * Update version to 2.0.0 * Update version to 2.0.0 * Update CHANGELOG.md * Add information on deepcave to the readme * Correct doc version * fix(runhistory): set id of the sampled config (#951) * fix: Set config_id of config in runhistory * ci: Set dependabot to target development branch (#963) * feat: Option preventing SMAC setting up logging (#947) * Version 2.0.0 (#898) * Update runhistory2epm.py * pre_commit * Updated changelog * Fixed comment * Updated changelog * Some small improvements * Removed trailing space * Updated changelog * Restricted pynisher * Updated changelog * Link JMLR paper * Update bibtex entry Before: arxiv paper, now journal article. * Fixed citation in docs * Fixed callback typos * BOinG for mixed hyperparameter search space * fix format * maint * maint documentation * maint pre-commit * fix workflow * maint * Add dependabot for workflow files * Bump actions/setup-python from 2 to 4 (#849) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/checkout from 2 to 3 (#850) Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * maint * Updated stale bot * Updated stale options again * Fixed typing * Increased version * rename check_point_in_ss to check_subspace_point * Make format * Uncomment test procedure * docstrings for boing facade * avoid typing namespace everywhere in epm_configuration_chooser * fix docstring in rh2epm_boing * Limit on dask * Trying to fix tests * Replaced typing in all files * docstring for boing subspace * Changed feature requests to feature * allow acq_optimizer_local_kwargs to be passed to boing subspace without acq_optimizer_local * fix pre-commit * Added any * Update facade * Fix typing * Match arguments from pSMAC with SMAC4AC signature * Add description of pSMAC to docstring * Add option to pass facade class to pSMAC * Add example for pSMAC * Removed docstring in readme's example * Corrected text * move construct_gp_kernels to smac.epm_gpytorch.boing_kernels * maint * fix wrongly imported func name in tests * Add reference for pSMAC #609 * Rename example file * Added roadmap to readme * Prettification * Add result merger * Reformat with black * Delete unnecessary import * Fix typing * Inject first trajectory entry from file from first rundir * Add test for ResultMerger * Fix docstring * Get runhistory and trajectory from pSMAC run Via the result merger. Searchers rundirs and merges found runhistories and calculates a new trajectory. * Get trajectory * Add regex to requirements * Fixed MO cache problem * Fix MO tests * Fixing MO in AbstractRacer * Accepting ints * Hacking in the sum cost for MO * Fix mypi * Bugfix * Make mypi happy * Fix tests * Make format * Remove num_obj in MO optimizer * A lot of restructuring * Solve importings * Solve importings * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Fixing, fixing, fixing, ... * Updated changelog * Still fixing :) * fix docstrings * Change directory of tests * Fix ambiguous variable * Del newlines * fix mypy * mypy and doc fix * Removed int type checking * fix tests * Always return List of incs * fix path issue in example * maint boing facade * Better docstring * Return values if no bounds passed * Removed import * Return only one incumbent * Rename n_optimizers -> n_workers * Update example * Update docstring * Remove legacy function * Update to new signature * Update changelog.md * Format black * Update test * Please pre-commit * Merge development * Fix tests * Fix typos * Format black * Updated text * Increased version * Improved readme * SMAC logo 50% * SMAC logo 50% * Removed dev-2.0 * Added doc workflow for latest tag * Fixing docs latest tag * Fixing docs latest tag * First steps new intensifier * Make it work again * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * Test issues * FIX formatting * Link to old documentation * MOVE random weight initialization to smbo ask method * ADD enable parego in unittests * FIX formatting * Fix unit tests * Add annotation * corrected log prob for tophat prior * Removed configurations from callback * Fixed tests * Fix mypy * Save docs as latest version too * Enabled all tests again * Make version an env * Added last version to docs * Global variable fix * Added v before version number * New Intensifier Implementation (#888) - Completely reimplemented the intensifiers (including Successive Halving and Hyperband): All intensifiers support multi-fidelity, multi-objective and multi-threading by nature now. - Expected behaviour for ask-and-tell interface ensured (also for Successive Halving). - Continuing a run is now fully supported. - Added more examples. - Updated documentation based on new implementation. * First benchmark steps * Added a warning * Set xi to 0.01 * Incorporated walltime in trajectory * Working version of benchmark * Changed requirements * Changed xi back to 0.0 again * No duplicate budgets * Fixed bug which updated tracker wrongly * Bugfix: Delete correct bracket+stage * Fixed tests * Added costs to the trajectory * Changed initial design max ratio to 0.25 * Make format * Added more details to intensifier * Finished benchmark * Added benchmark source files * Fixed tests because of initial design changes * Improved runhistory API * Fixed typo (#894) * Merge doc fixes * Doc fixes (#893) Co-authored-by: timruhkopf Co-authored-by: Carolin Benjamins Co-authored-by: dengdifan Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs * Make format * Satisfied mypy * Enabled more tests * Increased version * Mypy * Mypy * Mypy * Fixed critical bug when using highest budget for inc selection * Added SH+HB comparison * Stuff * Updated report * Fixed docstring * Fixed image path * Removed vscode folder * Bugfixes * Doc fixes * Updated changelog * Make isort * Fix typos * Removed new line * Added alpha2 to docs selection * update description * Fix 531 correctly create integers in initial design This PR fixes a bug in which integer hyperparameters would not be created correctly in the initial design. This went unnoticed, except in the case where it was part of a condition and the function deactivate_inactive_hyperparameters was called. This PR fixes this bug by mapping the random float that was sampled by, for example and LHD or Sobol sequence, to the float representation of an integer by calling _transform and _inverse_transform of the integer hyperparameter. * Add contact paragraph to README. * Increase days until stale issues are closed by stalebot (#904) * Edit changelog * Add issue number * Closes #907 * Clarify origin of configurations (#908) * Random forest speed up (#903) Co-authored-by: René Sass * Add: workflow to update pre-commit versions (#874) Co-authored-by: René Sass * Add Git-Flow Workflow Specification * [DOC] acquisition function maximizers * Add stale reminder email action * Increase version number of ConfigSpace * Add guideline for updating copyright notice years to CONTRIBUTING.md * Update requirement for pyrfr (#920) * Update pre-commit: isort,flake8 version numbers * Format black * Fix mypy * Ac bench (#929) * Add Branin AC Task * Add ac optimization type * Nicer plotting * Add ac mode * Update ac task * Make new smac env name adjustable * Increase python version for smac 1.4 * Update benchmark * Update plotting * Goodby comments * Fix path * Fix 1.4 wrapper * Update benchmark results --------- Co-authored-by: Carolin Benjamins * [DOC] added documentation on the incumbent update * Feature/add version to make publish (#931) * Add version, correct package name and use bash in makefile * Increase version * Increase version in makefile, add places with versions to make publish * Fix typo * Transform integer feature names to strings (#930) * Transform int features to str * Fix pre-commit * Update version to 2.0.0 * Update version to 2.0.0 * Update CHANGELOG.md * Add information on deepcave to the readme * Correct doc version --------- Co-authored-by: dengdifan <33290713+dengdifan@users.noreply.github.com> Co-authored-by: dengdifan Co-authored-by: Matthias Feurer Co-authored-by: Carolin Benjamins Co-authored-by: eddiebergman Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Tim Ruhkopf * feat: Option preventing SMAC setting up logging * fix: Run isort * Bump peter-evans/create-pull-request from 3 to 5 (#958) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 3 to 5. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v3...v5) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/setup-python from 2 to 4 (#945) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat: Option preventing SMAC setting up logging * fix: Run isort * fix: Add to changelog * feat: Option preventing SMAC setting up logging * fix: Run isort --------- Signed-off-by: dependabot[bot] Co-authored-by: René Sass Co-authored-by: dengdifan <33290713+dengdifan@users.noreply.github.com> Co-authored-by: dengdifan Co-authored-by: Matthias Feurer Co-authored-by: Carolin Benjamins Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Tim Ruhkopf * Citation update (#961) * update CITATION.cff * update order in CITATION.cff * fix citation date * fix citation * maint * Adjust hyperband configuration distribution across brackets * Compute the actual differences between the isb keys. (#957) * Compute the actual differences between the isb keys. * Added change to CHANGELOG.md * Adjust schedule for stale reminder * fix(logging): Prevent automatic logging setup at init (#970) * fix(logging): Prevent automatic logging setup at init * fix(logging): Rely on `setup_logging` to handle arg * Fix validate method of smbo, update docstring. * Allow callbacks to be added to a specific index, make callback registering public * Update CHANGELOG.md * Fix broken references (#988) * Adapt developer install instructions to include pre-commit installation (#994) * [Feature] Initial Design now supports executing the default config provided by the user by means of the configspace he generated. (and is no longer ignored) * Pipe dask_client through facade for exec on hpc (#983) * Pipe dask_client through facade for exec on hpc * Handle pynisher's memory arg Before: Could only pass int via SMAC with the default unit B, but now tuple/list is possible * Update CHANGELOG.md * Make format * No printi --------- Co-authored-by: Carolin Benjamins * [Bug-fix] Pr was failing due to mutable additional configs default * [doc-fix] the count of initial design configs is computed slightly differently * Feature/metadata callback (#999) * Add example for using a callback to log run metadata to a file * Add issue number in changelog * Changelog formatting * Simplify metadata example * Simplify metadata example * Put callback in smac directory to allow importing it in custom repos * Move callback files to callback directory * Edit changelog * Correct metadata callback argument types * Additional metadata arguments in example * Set metadata default values in callback * Edit comment to fix PR * Documentation/dask client example (#1001) * Update requirements (dask_jobqueue) * Fix broken link * Ignore parallel example * Add parallelization example * Remove imports, add comment abt dask client * Add E's information * Update docs * Add comment about n_workers --------- Co-authored-by: Carolin Benjamins * Documentation/dask client example (#1001) * Update requirements (dask_jobqueue) * Fix broken link * Ignore parallel example * Add parallelization example * Remove imports, add comment abt dask client * Add E's information * Update docs * Add comment about n_workers --------- Co-authored-by: Carolin Benjamins * Feature/dask client (#1002) * Add info message that scenario.n_workers is ignored When passing a dask client, directly set n_workers in client. * Change info to warning --------- Co-authored-by: Carolin Benjamins * Update smac version (#1003) * Update smac version * Add hint on github release in make publish * Update copyright year * check if config in rh when storing state (#997) * check if config in rh when storing state * maint * remove unnecessary changes * Adapt developer install instructions to include pre-commit installation (#994) * Pipe dask_client through facade for exec on hpc (#983) * Pipe dask_client through facade for exec on hpc * Handle pynisher's memory arg Before: Could only pass int via SMAC with the default unit B, but now tuple/list is possible * Update CHANGELOG.md * Make format * No printi --------- Co-authored-by: Carolin Benjamins * update changelog * Increase tolerance for runhistory restoration test --------- Co-authored-by: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Carolin Benjamins Co-authored-by: Helena Graf * Add a workaround to be able to pass a dataset via dask.scatter (#993) * [feat] Support dask.scatter * [fix] Add shared_data arg in abstract_facade * Address the comments by Eddie * [fix] Fix errors raised by Eddie * [format] Apply black locally * [fix] Get retries back to 16 as it was a mistake * [doc] Update CHANGELOG --------- Co-authored-by: helegraf Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> * Adding Binder links for example (#976) * binder * maint * fix dependencies * maint docs * Revert "maint docs" This reverts commit 4fa1a0fae8dfa0b646e43273d50f72ddd76bf191. * add requirement --------- Co-authored-by: Difan Deng * Remove commit message from dependabot PRs (#1009) --------- Signed-off-by: dependabot[bot] Co-authored-by: dengdifan Co-authored-by: René Sass Co-authored-by: Carolin Benjamins Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Matthias Feurer Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Eddie Bergman Co-authored-by: Tim Ruhkopf Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> Co-authored-by: Aron Bahram Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Co-authored-by: Difan Deng * Fix broken link --------- Signed-off-by: dependabot[bot] Co-authored-by: dengdifan Co-authored-by: René Sass Co-authored-by: Carolin Benjamins Co-authored-by: Carolin Benjamins Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> Co-authored-by: Katharina Eggensperger Co-authored-by: Matthias Feurer Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: Jeroen Rook Co-authored-by: Eddie Bergman Co-authored-by: Tim Ruhkopf Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> Co-authored-by: Aron Bahram Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Co-authored-by: Difan Deng * Drop torch requirements (#1015) * Remove torch requirements * Fix format --------- Co-authored-by: Carolin Benjamins * Remove leftover CLI entry points (#1014) * Fix incumbent selection in case of SH+MO * Update CHANGELOG.md * Feature/GitHub actions pr draft (#1008) Update the workflows only to trigger extensive tests for PRs that are not in the draft stage --------- Co-authored-by: timruhkopf Co-authored-by: Helena Graf * Update stale report to include issues with recent interactions (#1025) * Add recently updated issues to list of reminders. * Add recently updated issues to list of reminders. * Add recently updated issues to list of reminders. * Raise error if resource limitation and parallelization is requested (#1023) * Add warnings for parallelism doc * Check for parallelization + pynisher Does not work together, throw error. Does not work bc Dask pickles jobs and pynisher runs the target function in a process. * Fix whitespaces --------- Co-authored-by: Carolin Benjamins * Raise an error for invalid scatter data (#1018) * [enhance] Add an error for scatter data * Update ChangeLog --------- Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> * fix(configspace): shows all keys in print (#1043) * Fix callback order (#1040) Co-authored-by: Carolin Benjamins * fix: dtypes (#1044) * Fix(config_selector): Budget checking (#1039) * Fix: target runner with partial func (#1045) * Feature/improve documentation (#1028) * Experimental instruction for installing SMAC in windows via WSL * More detailed documentation regarding continuing runs * Fixes * Fixes * Fixes * Update docs/10_experimental.rst Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> * Fixes from PR * Fixes from PR * Fixes from PR * Fixes from PR * Fixes from PR --------- Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> * Handle configspace as dictionary in mlp example (#1057) * Handle configspace as dictionary in mlp example * Adapt sgd loss to newest scikit-learn version * Add change to changelog * Handle configspace as dictionary in parego example * Correct get usage * Raise version number * Add missing period in docs. * allow rf to impute OrdinalHyperparameter (#1065) * allow rf to impute OrdinalHyperparameter * Add missing period in docs. * Add issue number in changelog. --------- Co-authored-by: Sarah Krebs * fix dask_scheduler_file path (#1055) * fix dask_scheduler_file path * update change log --------- Co-authored-by: dengdifan * Fix config rejection for #1068 (#1069) * racing: make sure config only deleted from rejected configs if among incumbents upon update * test case for incumbent rejection * update code comments * added missing newline ahead of new test case * update change log --------- Co-authored-by: dengdifan Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> * fix(RandomForest): Always cast `seed` to `int` (#1084) * fix(RandomForest): Always cast `seed` to `int` In cases where seeds were generated from some numpy objects, sometimes you'd get back an `np.integer`, which causes the program to crash when communicating with `pyrfr` through `swig`. It seems that swig doesn't know that it's an _int-like_ and so we explicitly cast it to `int`. * Update CHANGELOG.md --------- Co-authored-by: C. Benjamins <75323339+benjamc@users.noreply.github.com> * Bump actions/checkout from 3 to 4 (#1072) * Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Update CHANGELOG.md --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: benjamc * chore: log warn on condition when custom dask client is provided (#1071) * Bump conda-incubator/setup-miniconda from 2 to 3 (#1087) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 2 to 3. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v2...v3) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Propagate the Scenario random seed to get_random_design (#1066) * Propagate the Scenario's random seed to get_random_design * add changelog line --------- Co-authored-by: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> * [#1056] Add example on intensify for cross-validation. (#1061) * Add example on intensify for cross-validation. * Improve example * Address Difan's comment * Update CHANGELOG.md * Change stale-reminder to recent-reminder (#1096) * Change stale-reminder to recent-reminder (#1095) (#1099) * Dont use mutable lists as default args (#1097) * Don't use mutable default argument. If this isn't respected, then additional_configs can be built up overtime when multiple optimizers are created in a single process. This can occur when the use_default_config option is set True and/or when additional_configs are provided. When different ConfigSpaces are provided this can result in an error due to mismatched defaults, for instance. In other cases it can result in non-deterministic behavior due to the random ordering of the configs that get added (e.g., with different parallel pytest runs). - [ ] Cleanup other sources of this error as well elsewhere in the code. - [ ] Add pylint checks for this issue. * fixup some others mutable defaults * add pylint checks * fix more related pylint issues * add changelog line * Update pre-commit config * Update pre-commit config * Remove TODO formulation as an issue already exists for this --------- Co-authored-by: Brian Kroth * Bump actions/setup-python from 4 to 5 (#1089) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump peter-evans/create-pull-request from 5 to 6 (#1093) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 5 to 6. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v5...v6) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * test: Fix upper bounds for `test_transformer()` and no random in `test_transformer_conditional()` (#1102) * test: Don't rely on randomization for the test * test: Include import * test: Fix boundaries for catgoricals in test * test: Missing import * doc: Make todo not more clear about when to upgrade * Change stale-reminder to recent-reminder (#1095) (#1107) * Fix typo in docstring for `MultiFideltyFacade.get_initial_design` (#1104) * Fix typo in docstring for `MultiFideltyFacade.get_initial_design` The docstring specifies 0.1 is used as a default value for max_ratio, however in code 0.25 is used. * Bug/acquisition maximizer sampling (#1106) * Adapt how n_points works in the config_selector.py Also remove LocalAndSortedPriorRandomSearch by integrating it into LocalAndSortedRandomSearch * Change retrain_after to 1 for BlackBOx Facade * Fix bugs in local_and_random_search.py * Edit CHANGELOG.md * fix acq maximizer docstring * call get config selector from super class * Update random and local search to new functionality * Update CHANGELOG.md * Fix sampling of initial points for local search when less previous configs than n_points * Re-format blackbo… * Create FUNDING.yml (#1113) * feat(hyperband): specify total budget instead of n_trials If you want to specify a total optimization budget in terms of fidelity units. In Hyperband, normally SMAC calculates a typical Hyperband round. If the number of trials is not used up by one single round, the next round is started. Instead of specifying the number of trial beforehand, specify the total budget in terms of the fidelity units and let SMAC calculate how many trials that would be. * make format * fix(style): codefactor * Update CHANGELOG.md --------- Signed-off-by: dependabot[bot] Co-authored-by: helegraf Co-authored-by: René Sass Co-authored-by: dengdifan Co-authored-by: Carolin Benjamins Co-authored-by: Katharina Eggensperger Co-authored-by: Matthias Feurer Co-authored-by: Eric Kalosa-Kenyon Co-authored-by: Pascal <66594767+PascalIversen@users.noreply.github.com> Co-authored-by: helegraf Co-authored-by: timruhkopf Co-authored-by: Alexander Tornede Co-authored-by: Sarah Krebs Co-authored-by: Alexander Tornede Co-authored-by: Alexander Tornede <44395781+alexandertornede@users.noreply.github.com> Co-authored-by: Sarah Krebs <30651149+sarah-krebs@users.noreply.github.com> Co-authored-by: Jeroen Rook Co-authored-by: Eddie Bergman Co-authored-by: Tim Ruhkopf Co-authored-by: Difan Deng <33290713+dengdifan@users.noreply.github.com> Co-authored-by: Aron Bahram Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sarah Segel <30651149+sarah-segel@users.noreply.github.com> Co-authored-by: Shuhei Watanabe <47781922+nabenabe0928@users.noreply.github.com> Co-authored-by: Difan Deng Co-authored-by: Bastian Zimmermann <10774221+BastianZim@users.noreply.github.com> Co-authored-by: Weihuang Wen Co-authored-by: PhilippBordne <71140732+PhilippBordne@users.noreply.github.com> Co-authored-by: SimonThormeyer <49559340+SimonThormeyer@users.noreply.github.com> Co-authored-by: Brian Kroth Co-authored-by: Brian Kroth Co-authored-by: fleance Co-authored-by: Marius Lindauer --- CHANGELOG.md | 1 + .../3_specify_HB_via_total_budget.py | 112 +++++++++++ smac/intensifier/hyperband_utils.py | 181 ++++++++++++++++++ 3 files changed, 294 insertions(+) create mode 100644 examples/2_multi_fidelity/3_specify_HB_via_total_budget.py create mode 100644 smac/intensifier/hyperband_utils.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 46b37206e..c66a8ef73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Dependencies - Update numpy NaN (#1122) and restrict numpy and ConfigSpace versions +- Add example to specify total budget (fidelity units) instead of n_trials for multi-fidelity/Hyperband (#1121) # 2.1.0 diff --git a/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py b/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py new file mode 100644 index 000000000..492cd7f46 --- /dev/null +++ b/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py @@ -0,0 +1,112 @@ +""" +Specify Number of Trials via a Total Budget in Hyperband +^^^^^^^^^^^^^^^^^^ +This example uses a dummy function but illustrates how to setup Hyperband if you +want to specify a total optimization budget in terms of fidelity units. + +In Hyperband, normally SMAC calculates a typical Hyperband round. +If the number of trials is not used up by one single round, the next round is started. +Instead of specifying the number of trial beforehand, specify the total budget +in terms of the fidelity units and let SMAC calculate how many trials that would be. + + +""" +from __future__ import annotations + +import numpy as np +from ConfigSpace import Configuration, ConfigurationSpace, Float +from matplotlib import pyplot as plt + +from smac import MultiFidelityFacade, RunHistory, Scenario +from smac.intensifier.hyperband_utils import get_n_trials_for_hyperband_multifidelity + +__copyright__ = "Copyright 2021, AutoML.org Freiburg-Hannover" +__license__ = "3-clause BSD" + + +class QuadraticFunction: + max_budget = 500 + + @property + def configspace(self) -> ConfigurationSpace: + cs = ConfigurationSpace(seed=0) + x = Float("x", (-5, 5), default=-5) + cs.add_hyperparameters([x]) + + return cs + + def train(self, config: Configuration, seed: int = 0, budget: float | None = None) -> float: + """Returns the y value of a quadratic function with a minimum we know to be at x=0.""" + x = config["x"] + + if budget is None: + multiplier = 1 + else: + multiplier = 1 + budget / self.max_budget + + return x**2 * multiplier + + +def plot(runhistory: RunHistory, incumbent: Configuration) -> None: + plt.figure() + + # Plot ground truth + x = list(np.linspace(-5, 5, 100)) + y = [xi * xi for xi in x] + plt.plot(x, y) + + # Plot all trials + for k, v in runhistory.items(): + config = runhistory.get_config(k.config_id) + x = config["x"] + y = v.cost # type: ignore + plt.scatter(x, y, c="blue", alpha=0.1, zorder=9999, marker="o") + + # Plot incumbent + plt.scatter(incumbent["x"], incumbent["x"] * incumbent["x"], c="red", zorder=10000, marker="x") + + plt.show() + + +if __name__ == "__main__": + model = QuadraticFunction() + + min_budget = 10 # minimum budget per trial + max_budget = 500 # maximum budget per trial + eta = 3 # standard HB parameter influencing the number of stages + + # Let's calculate how many trials we need to exhaust the total optimization budget (in terms of + # fidelity units) + n_trials = get_n_trials_for_hyperband_multifidelity( + total_budget=10000, # this is the total optimization budget we specify in terms of fidelity units + min_budget=min_budget, # This influences the Hyperband rounds, minimum budget per trial + max_budget=max_budget, # This influences the Hyperband rounds, maximum budget per trial + eta=eta, # This influences the Hyperband rounds + print_summary=True, + ) + + # Scenario object specifying the optimization "environment" + scenario = Scenario( + model.configspace, deterministic=True, n_trials=n_trials, min_budget=min_budget, max_budget=max_budget + ) + + # Now we use SMAC to find the best hyperparameters + smac = MultiFidelityFacade( + scenario, + model.train, # We pass the target function here + overwrite=True, # Overrides any previous results that are found that are inconsistent with the meta-data + intensifier=MultiFidelityFacade.get_intensifier(scenario=scenario, eta=eta), + ) + + incumbent = smac.optimize() + + # Get cost of default configuration + default_cost = smac.validate(model.configspace.get_default_configuration()) + print(f"Default cost: {default_cost}") + + # Let's calculate the cost of the incumbent + incumbent_cost = smac.validate(incumbent) + print(f"Incumbent cost: {incumbent_cost}") + + # Let's plot it too + plot(smac.runhistory, incumbent) diff --git a/smac/intensifier/hyperband_utils.py b/smac/intensifier/hyperband_utils.py new file mode 100644 index 000000000..ad8773a71 --- /dev/null +++ b/smac/intensifier/hyperband_utils.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import numpy as np + +from smac.intensifier.successive_halving import SuccessiveHalving + + +def determine_HB(min_budget: float, max_budget: float, eta: int = 3) -> dict: + """Determine one Hyperband round + + Parameters + ---------- + min_budget : float + Minimum budget per trial in fidelity units + max_budget : float + Maximum budget per trial in fidelity units + eta : int, defaults to 3 + Input that controls the proportion of configurations discarded in each round of Successive Halving. + + Returns + ------- + dict + Info about the Hyperband round + "max_iterations" + "n_configs_in_stage" + "budgets_in_stage" + "trials_used" + "budget_used" + "number_of_brackets" + + """ + _s_max = SuccessiveHalving._get_max_iterations(eta, max_budget, min_budget) + + _max_iterations: dict[int, int] = {} + _n_configs_in_stage: dict[int, list] = {} + _budgets_in_stage: dict[int, list] = {} + + for i in range(_s_max + 1): + max_iter = _s_max - i + + _budgets_in_stage[i], _n_configs_in_stage[i] = SuccessiveHalving._compute_configs_and_budgets_for_stages( + eta, max_budget, max_iter, _s_max + ) + _max_iterations[i] = max_iter + 1 + + total_trials = np.sum([np.sum(v) for v in _n_configs_in_stage.values()]) + total_budget = np.sum([np.sum(v) for v in _budgets_in_stage.values()]) + + return { + "max_iterations": _max_iterations, + "n_configs_in_stage": _n_configs_in_stage, + "budgets_in_stage": _budgets_in_stage, + "trials_used": total_trials, + "budget_used": total_budget, + "number_of_brackets": _s_max, + } + + +def determine_hyperband_for_multifidelity( + total_budget: float, min_budget: float, max_budget: float, eta: int = 3 +) -> dict: + """Determine how many Hyperband rounds should happen based on a total budget + + Parameters + ---------- + total_budget : float + Total budget for the complete optimization in fidelity units + min_budget : float + Minimum budget per trial in fidelity units + max_budget : float + Maximum budget per trial in fidelity units + eta : int, defaults to 3 + Input that controls the proportion of configurations discarded in each round of Successive Halving. + + Returns + ------- + dict + Info about the Hyperband round + "max_iterations" + "n_configs_in_stage" + "budgets_in_stage" + "trials_used" + "budget_used" + "number_of_brackets" + + """ + # Determine the HB + hyperband_round = determine_HB(eta=eta, min_budget=min_budget, max_budget=max_budget) + + # Calculate how many HB rounds we can have + budget_used_per_hyperband_round = hyperband_round["budget_used"] + number_of_full_hb_rounds = int(np.floor(total_budget / budget_used_per_hyperband_round)) + remaining_budget = total_budget % budget_used_per_hyperband_round + trials_used_per_hb_round = hyperband_round["trials_used"] + n_configs_in_stage = hyperband_round["n_configs_in_stage"] + budgets_in_stage = hyperband_round["budgets_in_stage"] + + remaining_trials = 0 + for stage in n_configs_in_stage.keys(): + B = budgets_in_stage[stage] + C = n_configs_in_stage[stage] + for b, c in zip(B, C): + # How many trials are left? + # If b * c is lower than remaining budget, we can add full c + # otherwise we need to find out how many trials we can do with this budget + remaining_trials += min(c, int(np.floor(remaining_budget / b))) + # We cannot go lower than 0 + # If we are in the case of b*c > remaining_budget, we will not have any + # budget left. We can not add full c but the number of trials that still fit + remaining_budget = max(0, remaining_budget - b * c) + + # print(stage, b, c) + # print("-"*20, remaining_trials, remaining_budget) + + n_trials = int(number_of_full_hb_rounds * trials_used_per_hb_round + remaining_trials) + + hyperband_info = hyperband_round + hyperband_info["n_trials"] = n_trials + hyperband_info["total_budget"] = total_budget + hyperband_info["eta"] = eta + hyperband_info["min_budget"] = min_budget + hyperband_info["max_budget"] = max_budget + + return hyperband_info + + +def print_hyperband_summary(hyperband_info: dict) -> None: + """Print summary about Hyperband as used in the MultiFidelityFacade + + Parameters + ---------- + hyperband_info : dict + Info dict about Hyperband + """ + print("-" * 30, "HYPERBAND IN MULTI-FIDELITY", "-" * 30) + print("total budget:\t\t", hyperband_info["total_budget"]) + print("total number of trials:\t", hyperband_info["n_trials"]) + print("number of HB rounds:\t", hyperband_info["total_budget"] / hyperband_info["budget_used"]) + print() + + print("\t~~~~~~~~~~~~HYPERBAND ROUND") + print("\teta:\t\t\t\t\t", hyperband_info["eta"]) + print("\tmin budget per trial:\t\t\t", hyperband_info["min_budget"]) + print("\tmax budget per trial:\t\t\t", hyperband_info["max_budget"]) + print("\ttotal number of trials per HB round:\t", hyperband_info["trials_used"]) + print("\tbudget used per HB round:\t\t", hyperband_info["budget_used"]) + print("\tnumber of brackets:\t\t\t", hyperband_info["number_of_brackets"]) + print("\tbudgets per stage:\t\t\t", hyperband_info["budgets_in_stage"]) + print("\tn configs per stage:\t\t\t", hyperband_info["n_configs_in_stage"]) + print("-" * (2 * 30 + len("HYPERBAND IN MULTI-FIDELITY") + 2)) + + +def get_n_trials_for_hyperband_multifidelity( + total_budget: float, min_budget: float, max_budget: float, eta: int = 3, print_summary: bool = True +) -> int: + """Caculate the number of trials needed for multi-fidelity optimization + + Specify the total budget and find out how many trials that equals. + + Parameters + ---------- + total_budget : float + Total budget for the complete optimization in fidelity units + min_budget : float + Minimum budget per trial in fidelity units + max_budget : float + Maximum budget per trial in fidelity units + eta : int, defaults to 3 + Input that controls the proportion of configurations discarded in each round of Successive Halving. + + Returns + ------- + int + Number of trials needed for the specified total budgets + """ + hyperband_info = determine_hyperband_for_multifidelity( + total_budget=total_budget, eta=eta, min_budget=min_budget, max_budget=max_budget + ) + if print_summary: + print_hyperband_summary(hyperband_info=hyperband_info) + return hyperband_info["n_trials"] From 88bc2f7546e2b374e3f16bd2243e553b67736b0f Mon Sep 17 00:00:00 2001 From: "C. Benjamins" <75323339+benjamc@users.noreply.github.com> Date: Tue, 16 Jul 2024 17:52:05 +0200 Subject: [PATCH 40/57] Update CHANGELOG.md --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c66a8ef73..b209815d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,10 @@ # 2.1.1 +## Features +- Add example to specify total budget (fidelity units) instead of n_trials for multi-fidelity/Hyperband (#1121) + ## Dependencies - Update numpy NaN (#1122) and restrict numpy and ConfigSpace versions -- Add example to specify total budget (fidelity units) instead of n_trials for multi-fidelity/Hyperband (#1121) # 2.1.0 From 73ff8d0b822feb00c939383a08b536aa67acdd97 Mon Sep 17 00:00:00 2001 From: Difan Deng <33290713+dengdifan@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:33:31 +0200 Subject: [PATCH 41/57] Revert 1023 (#1123) * Revert "Raise error if resource limitation and parallelization is requested (#1023)" This reverts commit cb1cd2f6436bc1dd5e963d6a4fe3f9efed4fcf35. * update examples * update daks_queue version --- docs/advanced_usage/9_parallelism.rst | 33 ------------------- .../1_basics/7_parallelization_cluster.py | 9 ++++- setup.py | 2 +- smac/facade/abstract_facade.py | 12 ------- 4 files changed, 9 insertions(+), 47 deletions(-) diff --git a/docs/advanced_usage/9_parallelism.rst b/docs/advanced_usage/9_parallelism.rst index 67e555215..9912c782f 100644 --- a/docs/advanced_usage/9_parallelism.rst +++ b/docs/advanced_usage/9_parallelism.rst @@ -21,39 +21,6 @@ SMAC supports multiple workers natively via Dask. Just specify ``n_workers`` in When using multiple workers, SMAC is not reproducible anymore. -.. warning :: - - You cannot use resource limitation (pynisher, via the `scenario` arguments `trail_walltime_limit` and `trial_memory_limit`). - This is because pynisher works by running your function inside of a subprocess. - Once in the subprocess, the resources will be limited for that process before running your function. - This does not work together with pickling - which is required by dask to schedule jobs on the cluster, even on a local one. - - -.. warning :: - - Start/run SMAC inside ``if __name__ == "__main__"`` in your script otherwise Dask is not able to correctly - spawn jobs and probably this runtime error will be raised: - - .. code-block :: - - RuntimeError: - An attempt has been made to start a new process before the - current process has finished its bootstrapping phase. - - This probably means that you are not using fork to start your - child processes and you have forgotten to use the proper idiom - in the main module: - - if __name__ == '__main__': - freeze_support() - ... - - The "freeze_support()" line can be omitted if the program - is not going to be frozen to produce an executable. - - - - Running on a Cluster -------------------- You can also pass a custom dask client, e.g. to run on a slurm cluster. diff --git a/examples/1_basics/7_parallelization_cluster.py b/examples/1_basics/7_parallelization_cluster.py index 36f79586e..1a6167669 100644 --- a/examples/1_basics/7_parallelization_cluster.py +++ b/examples/1_basics/7_parallelization_cluster.py @@ -6,6 +6,9 @@ SLURM cluster. If you do not want to use a cluster but your local machine, set dask_client to `None` and pass `n_workers` to the `Scenario`. +Sometimes, the submitted jobs by the slurm client might be cancelled once it starts. In that +case, you could try to start your job from a computing node + :warning: On some clusters you cannot spawn new jobs when running a SLURMCluster inside a job instead of on the login node. No obvious errors might be raised but it can hang silently. @@ -77,7 +80,7 @@ def train(self, config: Configuration, seed: int = 0) -> float: model = Branin() # Scenario object specifying the optimization "environment" - scenario = Scenario(model.configspace, deterministic=True, n_trials=100) + scenario = Scenario(model.configspace, deterministic=True, n_trials=100, trial_walltime_limit=100) # Create cluster n_workers = 4 # Use 4 workers on the cluster @@ -97,6 +100,10 @@ def train(self, config: Configuration, seed: int = 0) -> float: walltime="00:10:00", processes=1, log_directory="tmp/smac_dask_slurm", + # if you would like to limit the resources consumption of each function evaluation with pynisher, you need to + # set nanny as False + # Otherwise, an error `daemonic processes are not allowed to have children` will raise! + nanny=False # if you do not use pynisher to limit the memory/time usage, feel free to set this one as True ) cluster.scale(jobs=n_workers) diff --git a/setup.py b/setup.py index 9e8467e4b..85324e17d 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ def read_file(filepath: str) -> str: "scikit-learn>=1.1.2", "pyrfr>=0.9.0", "dask[distributed]", - "dask_jobqueue", + "dask_jobqueue>=0.8.2", "emcee>=3.0.0", "regex", "pyyaml", diff --git a/smac/facade/abstract_facade.py b/smac/facade/abstract_facade.py index 6ca49b057..9a2031099 100644 --- a/smac/facade/abstract_facade.py +++ b/smac/facade/abstract_facade.py @@ -463,18 +463,6 @@ def _validate(self) -> None: # Make sure the same acquisition function is used assert self._acquisition_function == self._acquisition_maximizer._acquisition_function - if isinstance(self._runner, DaskParallelRunner) and ( - self.scenario.trial_walltime_limit is not None or self.scenario.trial_memory_limit is not None - ): - # This is probably due to pickling dask jobs - raise ValueError( - "Parallelization via Dask cannot be used in combination with limiting " - "the resources " - "of the target function via `scenario.trial_walltime_limit` or " - "`scenario.trial_memory_limit`. Set those to `None` if you want " - "parallelization. " - ) - def _get_signature_arguments(self) -> list[str]: """Returns signature arguments, which are required by the intensifier.""" arguments = [] From 03a2701efed3987af61d2a175b76789ef32db668 Mon Sep 17 00:00:00 2001 From: "C. Benjamins" <75323339+benjamc@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:56:39 +0200 Subject: [PATCH 42/57] Upgrade to ConfigSpace 1.x (#1124) * Upgrad to ConfigSpace 1.x * refactor: ConfigSpace API * fix(jsondump): add numpy encoder ConfigSpace integer hyperparameters do not return python int type but numpy int64, which is not json serializable * fix(jsondump): add numpy encoder * fix(ConfigSpace): parameter API * Fix example (ConfigSpace API) * Make format * Fix pre-commit * isort --- CHANGELOG.md | 3 +- benchmark/src/benchmark.py | 4 +- benchmark/src/models/ac_branin.py | 2 +- benchmark/src/models/branin.py | 2 +- benchmark/src/models/himmelblau.py | 2 +- benchmark/src/models/mlp.py | 4 +- benchmark/src/models/svm.py | 4 +- examples/1_basics/1_quadratic_function.py | 2 +- examples/1_basics/2_svm_cv.py | 4 +- examples/1_basics/3_ask_and_tell.py | 2 +- examples/1_basics/4_callback.py | 2 +- examples/1_basics/5_continue.py | 2 +- examples/1_basics/6_priors.py | 6 +- .../1_basics/7_parallelization_cluster.py | 2 +- examples/2_multi_fidelity/1_mlp_epochs.py | 4 +- examples/2_multi_fidelity/2_sgd_datasets.py | 2 +- .../3_specify_HB_via_total_budget.py | 2 +- examples/3_multi_objective/2_parego.py | 4 +- .../4_advanced_optimizer/1_turbo_optimizer.py | 2 +- .../4_advanced_optimizer/2_boing_optimizer.py | 2 +- .../3_metadata_callback.py | 2 +- .../4_intensify_crossvalidation.py | 27 +++---- setup.py | 2 +- .../maximizer/local_and_random_search.py | 1 - smac/acquisition/maximizer/local_search.py | 2 +- smac/callback/metadata_callback.py | 3 +- .../initial_design/abstract_initial_design.py | 1 - smac/intensifier/abstract_intensifier.py | 3 +- .../hyperband_budget_calculation.ipynb | 73 +++++++++++++++++ smac/intensifier/successive_halving.py | 2 +- smac/main/smbo.py | 3 +- .../gaussian_process/kernels/rbf_kernel.py | 1 - .../gaussian_process/kernels/white_kernel.py | 1 - .../gaussian_process/mcmc_gaussian_process.py | 1 - smac/runhistory/runhistory.py | 4 +- smac/scenario.py | 11 +-- smac/utils/configspace.py | 8 +- smac/utils/numpyencoder.py | 62 ++++++++++++++ smac/utils/subspaces/__init__.py | 2 +- tests/conftest.py | 1 - tests/fixtures/configspace.py | 6 +- tests/fixtures/models.py | 4 +- tests/test_acquisition/test_maximizers.py | 13 ++- .../test_initial_design/test_sobol_design.py | 2 +- .../test_abstract_intensifier.py | 27 ++----- tests/test_model/_test_gp_gpytorch.py | 1 - tests/test_model/test_gp.py | 3 - tests/test_model/test_rf.py | 1 - tests/test_runhistory/test_runhistory.py | 5 -- .../test_runhistory_encoder.py | 81 +++++-------------- 50 files changed, 239 insertions(+), 171 deletions(-) create mode 100644 smac/intensifier/hyperband_budget_calculation.ipynb create mode 100644 smac/utils/numpyencoder.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b209815d3..045253993 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ - Add example to specify total budget (fidelity units) instead of n_trials for multi-fidelity/Hyperband (#1121) ## Dependencies -- Update numpy NaN (#1122) and restrict numpy and ConfigSpace versions +- Update numpy NaN (#1122) and restrict numpy version +- Upgrade to ConfigSpace 1.x.x (#1124) # 2.1.0 diff --git a/benchmark/src/benchmark.py b/benchmark/src/benchmark.py index 55cabd6d4..9ae74a5fb 100644 --- a/benchmark/src/benchmark.py +++ b/benchmark/src/benchmark.py @@ -17,6 +17,8 @@ from collections import defaultdict from pathlib import Path +from smac.utils.numpyencoder import NumpyEncoder + import pandas as pd from src.tasks import TASKS # noqa: E402 from src.utils.exceptions import NotSupportedError # noqa: E402 @@ -79,7 +81,7 @@ def _save_data(self) -> None: """Saves the internal data to the file.""" print("Saving data...") with open(str(RAW_FILENAME), "w") as f: - json.dump(self._data, f, indent=4) + json.dump(self._data, f, indent=4, cls=NumpyEncoder) def _fill_keys(self) -> None: """Fill data with keys based on computer name, tasks, and selected version.""" diff --git a/benchmark/src/models/ac_branin.py b/benchmark/src/models/ac_branin.py index 12287520f..e55c2862a 100644 --- a/benchmark/src/models/ac_branin.py +++ b/benchmark/src/models/ac_branin.py @@ -20,7 +20,7 @@ def configspace(self) -> ConfigurationSpace: x2 = Float("x2", (0, 15), default=7.5) # Add hyperparameters and conditions to our configspace - cs.add_hyperparameters([x2]) + cs.add([x2]) return cs diff --git a/benchmark/src/models/branin.py b/benchmark/src/models/branin.py index 1fd20554f..0f86d82ac 100644 --- a/benchmark/src/models/branin.py +++ b/benchmark/src/models/branin.py @@ -20,7 +20,7 @@ def configspace(self) -> ConfigurationSpace: x2 = Float("x2", (0, 15), default=0) # Add hyperparameters and conditions to our configspace - cs.add_hyperparameters([x1, x2]) + cs.add([x1, x2]) return cs diff --git a/benchmark/src/models/himmelblau.py b/benchmark/src/models/himmelblau.py index cab99019a..c12029e1a 100644 --- a/benchmark/src/models/himmelblau.py +++ b/benchmark/src/models/himmelblau.py @@ -19,7 +19,7 @@ def configspace(self) -> ConfigurationSpace: y = Float("y", (-5, 5)) # Add hyperparameters and conditions to our configspace - cs.add_hyperparameters([x, y]) + cs.add([x, y]) return cs diff --git a/benchmark/src/models/mlp.py b/benchmark/src/models/mlp.py index 4867a2f42..7329de0df 100644 --- a/benchmark/src/models/mlp.py +++ b/benchmark/src/models/mlp.py @@ -33,7 +33,7 @@ def configspace(self) -> ConfigurationSpace: learning_rate_init = Float("learning_rate_init", (0.0001, 1.0), default=0.001, log=True) # Add all hyperparameters at once: - cs.add_hyperparameters([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) + cs.add([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) # Adding conditions to restrict the hyperparameter space... # ... since learning rate is used when solver is 'sgd'. @@ -44,7 +44,7 @@ def configspace(self) -> ConfigurationSpace: use_batch_size = InCondition(child=batch_size, parent=solver, values=["sgd", "adam"]) # We can also add multiple conditions on hyperparameters at once: - cs.add_conditions([use_lr, use_batch_size, use_lr_init]) + cs.add([use_lr, use_batch_size, use_lr_init]) return cs diff --git a/benchmark/src/models/svm.py b/benchmark/src/models/svm.py index 88159ab70..3ed294ad0 100644 --- a/benchmark/src/models/svm.py +++ b/benchmark/src/models/svm.py @@ -34,8 +34,8 @@ def configspace(self) -> ConfigurationSpace: use_gamma_value = InCondition(child=gamma_value, parent=gamma, values=["value"]) # Add hyperparameters and conditions to our configspace - cs.add_hyperparameters([kernel, C, shrinking, degree, coef, gamma, gamma_value]) - cs.add_conditions([use_degree, use_coef, use_gamma, use_gamma_value]) + cs.add([kernel, C, shrinking, degree, coef, gamma, gamma_value]) + cs.add([use_degree, use_coef, use_gamma, use_gamma_value]) return cs diff --git a/examples/1_basics/1_quadratic_function.py b/examples/1_basics/1_quadratic_function.py index 3cafb846b..4d27c7ae6 100644 --- a/examples/1_basics/1_quadratic_function.py +++ b/examples/1_basics/1_quadratic_function.py @@ -26,7 +26,7 @@ class QuadraticFunction: def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x = Float("x", (-5, 5), default=-5) - cs.add_hyperparameters([x]) + cs.add([x]) return cs diff --git a/examples/1_basics/2_svm_cv.py b/examples/1_basics/2_svm_cv.py index cfe39d4bb..345fcffb0 100644 --- a/examples/1_basics/2_svm_cv.py +++ b/examples/1_basics/2_svm_cv.py @@ -46,8 +46,8 @@ def configspace(self) -> ConfigurationSpace: use_gamma_value = InCondition(child=gamma_value, parent=gamma, values=["value"]) # Add hyperparameters and conditions to our configspace - cs.add_hyperparameters([kernel, C, shrinking, degree, coef, gamma, gamma_value]) - cs.add_conditions([use_degree, use_coef, use_gamma, use_gamma_value]) + cs.add([kernel, C, shrinking, degree, coef, gamma, gamma_value]) + cs.add([use_degree, use_coef, use_gamma, use_gamma_value]) return cs diff --git a/examples/1_basics/3_ask_and_tell.py b/examples/1_basics/3_ask_and_tell.py index 5d0e5e78c..6ab8b5ba8 100644 --- a/examples/1_basics/3_ask_and_tell.py +++ b/examples/1_basics/3_ask_and_tell.py @@ -20,7 +20,7 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x0 = Float("x0", (-5, 10), default=-3) x1 = Float("x1", (-5, 10), default=-4) - cs.add_hyperparameters([x0, x1]) + cs.add([x0, x1]) return cs diff --git a/examples/1_basics/4_callback.py b/examples/1_basics/4_callback.py index c3d66a4b9..0fd9e9d9d 100644 --- a/examples/1_basics/4_callback.py +++ b/examples/1_basics/4_callback.py @@ -27,7 +27,7 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x0 = Float("x0", (-5, 10), default=-3) x1 = Float("x1", (-5, 10), default=-4) - cs.add_hyperparameters([x0, x1]) + cs.add([x0, x1]) return cs diff --git a/examples/1_basics/5_continue.py b/examples/1_basics/5_continue.py index 025856fee..63cfb3957 100644 --- a/examples/1_basics/5_continue.py +++ b/examples/1_basics/5_continue.py @@ -47,7 +47,7 @@ class QuadraticFunction: def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x = Float("x", (-5, 5), default=-5) - cs.add_hyperparameters([x]) + cs.add([x]) return cs diff --git a/examples/1_basics/6_priors.py b/examples/1_basics/6_priors.py index 218bd8f46..691460c0b 100644 --- a/examples/1_basics/6_priors.py +++ b/examples/1_basics/6_priors.py @@ -95,13 +95,13 @@ def configspace(self) -> ConfigurationSpace: "learning_rate_init", lower=1e-5, upper=1.0, - mu=np.log(1e-3), - sigma=np.log(10), + mu=1e-3, # will be transformed to log space later + sigma=10, # will be transformed to log space later log=True, ) # Add all hyperparameters at once: - cs.add_hyperparameters([n_layer, n_neurons, activation, optimizer, batch_size, learning_rate_init]) + cs.add([n_layer, n_neurons, activation, optimizer, batch_size, learning_rate_init]) return cs diff --git a/examples/1_basics/7_parallelization_cluster.py b/examples/1_basics/7_parallelization_cluster.py index 1a6167669..e5c33f94d 100644 --- a/examples/1_basics/7_parallelization_cluster.py +++ b/examples/1_basics/7_parallelization_cluster.py @@ -44,7 +44,7 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x0 = Float("x0", (-5, 10), default=-5, log=False) x1 = Float("x1", (0, 15), default=2, log=False) - cs.add_hyperparameters([x0, x1]) + cs.add([x0, x1]) return cs diff --git a/examples/2_multi_fidelity/1_mlp_epochs.py b/examples/2_multi_fidelity/1_mlp_epochs.py index 48c027d3b..5cb0aefa0 100644 --- a/examples/2_multi_fidelity/1_mlp_epochs.py +++ b/examples/2_multi_fidelity/1_mlp_epochs.py @@ -65,7 +65,7 @@ def configspace(self) -> ConfigurationSpace: learning_rate_init = Float("learning_rate_init", (0.0001, 1.0), default=0.001, log=True) # Add all hyperparameters at once: - cs.add_hyperparameters([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) + cs.add([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) # Adding conditions to restrict the hyperparameter space... # ... since learning rate is only used when solver is 'sgd'. @@ -76,7 +76,7 @@ def configspace(self) -> ConfigurationSpace: use_batch_size = InCondition(child=batch_size, parent=solver, values=["sgd", "adam"]) # We can also add multiple conditions on hyperparameters at once: - cs.add_conditions([use_lr, use_batch_size, use_lr_init]) + cs.add([use_lr, use_batch_size, use_lr_init]) return cs diff --git a/examples/2_multi_fidelity/2_sgd_datasets.py b/examples/2_multi_fidelity/2_sgd_datasets.py index 09864a963..178ea21c2 100644 --- a/examples/2_multi_fidelity/2_sgd_datasets.py +++ b/examples/2_multi_fidelity/2_sgd_datasets.py @@ -76,7 +76,7 @@ def configspace(self) -> ConfigurationSpace: learning_rate = Categorical("learning_rate", ["constant", "invscaling", "adaptive"], default="constant") eta0 = Float("eta0", (0.00001, 1), default=0.1, log=True) # Add the parameters to configuration space - cs.add_hyperparameters([alpha, l1_ratio, learning_rate, eta0]) + cs.add([alpha, l1_ratio, learning_rate, eta0]) return cs diff --git a/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py b/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py index 492cd7f46..7c0ebdcf0 100644 --- a/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py +++ b/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py @@ -31,7 +31,7 @@ class QuadraticFunction: def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x = Float("x", (-5, 5), default=-5) - cs.add_hyperparameters([x]) + cs.add([x]) return cs diff --git a/examples/3_multi_objective/2_parego.py b/examples/3_multi_objective/2_parego.py index 856c2e857..b5294fb98 100644 --- a/examples/3_multi_objective/2_parego.py +++ b/examples/3_multi_objective/2_parego.py @@ -54,14 +54,14 @@ def configspace(self) -> ConfigurationSpace: learning_rate = Categorical("learning_rate", ["constant", "invscaling", "adaptive"], default="constant") learning_rate_init = Float("learning_rate_init", (0.0001, 1.0), default=0.001, log=True) - cs.add_hyperparameters([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) + cs.add([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) use_lr = EqualsCondition(child=learning_rate, parent=solver, value="sgd") use_lr_init = InCondition(child=learning_rate_init, parent=solver, values=["sgd", "adam"]) use_batch_size = InCondition(child=batch_size, parent=solver, values=["sgd", "adam"]) # We can also add multiple conditions on hyperparameters at once: - cs.add_conditions([use_lr, use_batch_size, use_lr_init]) + cs.add([use_lr, use_batch_size, use_lr_init]) return cs diff --git a/examples/4_advanced_optimizer/1_turbo_optimizer.py b/examples/4_advanced_optimizer/1_turbo_optimizer.py index 860243c02..dc936f707 100644 --- a/examples/4_advanced_optimizer/1_turbo_optimizer.py +++ b/examples/4_advanced_optimizer/1_turbo_optimizer.py @@ -28,7 +28,7 @@ # cs = ConfigurationSpace(seed=0) # x0 = Float("x0", (-5, 10), default=-3) # x1 = Float("x1", (-5, 10), default=-4) -# cs.add_hyperparameters([x0, x1]) +# cs.add([x0, x1]) # return cs diff --git a/examples/4_advanced_optimizer/2_boing_optimizer.py b/examples/4_advanced_optimizer/2_boing_optimizer.py index 3eb1a1869..815b1f340 100644 --- a/examples/4_advanced_optimizer/2_boing_optimizer.py +++ b/examples/4_advanced_optimizer/2_boing_optimizer.py @@ -27,7 +27,7 @@ # cs = ConfigurationSpace(seed=0) # x0 = Float("x0", (-5, 10), default=-3) # x1 = Float("x1", (-5, 10), default=-4) -# cs.add_hyperparameters([x0, x1]) +# cs.add([x0, x1]) # return cs diff --git a/examples/4_advanced_optimizer/3_metadata_callback.py b/examples/4_advanced_optimizer/3_metadata_callback.py index ab35f2862..b82670dbc 100644 --- a/examples/4_advanced_optimizer/3_metadata_callback.py +++ b/examples/4_advanced_optimizer/3_metadata_callback.py @@ -36,7 +36,7 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x0 = Float("x0", (-5, 10), default=-3) x1 = Float("x1", (-5, 10), default=-4) - cs.add_hyperparameters([x0, x1]) + cs.add([x0, x1]) return cs diff --git a/examples/4_advanced_optimizer/4_intensify_crossvalidation.py b/examples/4_advanced_optimizer/4_intensify_crossvalidation.py index d215dd8ec..679253da1 100644 --- a/examples/4_advanced_optimizer/4_intensify_crossvalidation.py +++ b/examples/4_advanced_optimizer/4_intensify_crossvalidation.py @@ -35,17 +35,17 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) # First we create our hyperparameters - C = Float("C", (2 ** - 5, 2 ** 15), default=1.0, log=True) - gamma = Float("gamma", (2 ** -15, 2 ** 3), default=1.0, log=True) + C = Float("C", (2**-5, 2**15), default=1.0, log=True) + gamma = Float("gamma", (2**-15, 2**3), default=1.0, log=True) # Add hyperparameters to our configspace - cs.add_hyperparameters([C, gamma]) + cs.add([C, gamma]) return cs def train(self, config: Configuration, instance: str, seed: int = 0) -> float: """Creates a SVM based on a configuration and evaluate on the given fold of the digits dataset - + Parameters ---------- config: Configuration @@ -81,15 +81,14 @@ def train(self, config: Configuration, instance: str, seed: int = 0) -> float: scenario = Scenario( classifier.configspace, n_trials=50, # We want to run max 50 trials (combination of config and instances in the case of - # deterministic=True. In the case of deterministic=False, this would be the - # combination of instances, seeds and configs). The number of distinct configurations - # evaluated by SMAC will be lower than this number because some of the configurations - # will be executed on more than one instance (CV fold). + # deterministic=True. In the case of deterministic=False, this would be the + # combination of instances, seeds and configs). The number of distinct configurations + # evaluated by SMAC will be lower than this number because some of the configurations + # will be executed on more than one instance (CV fold). instances=[f"{i}" for i in range(N_FOLDS)], # Specify all instances by their name (as a string) - instance_features={f"{i}": [i] for i in range(N_FOLDS)}, # breaks SMAC + instance_features={f"{i}": [i] for i in range(N_FOLDS)}, # breaks SMAC deterministic=True # To simplify the problem we make SMAC believe that we have a deterministic - # optimization problem. - + # optimization problem. ) # We want to run the facade's default initial design, but we want to change the number @@ -102,12 +101,12 @@ def train(self, config: Configuration, instance: str, seed: int = 0) -> float: classifier.train, initial_design=initial_design, overwrite=True, # If the run exists, we overwrite it; alternatively, we can continue from last state - # The next line defines the intensifier, i.e., the module that governs the selection of + # The next line defines the intensifier, i.e., the module that governs the selection of # instance-seed pairs. Since we set deterministic to True above, it only governs the instance in # this example. Technically, it is not necessary to create the intensifier as a user, but it is # necessary to do so because we change the argument max_config_calls (the number of instance-seed pairs # per configuration to try) to the number of cross-validation folds, while the default would be 3. - intensifier=Intensifier(scenario=scenario, max_config_calls=N_FOLDS, seed=0) + intensifier=Intensifier(scenario=scenario, max_config_calls=N_FOLDS, seed=0), ) incumbent = smac.optimize() @@ -124,4 +123,4 @@ def train(self, config: Configuration, instance: str, seed: int = 0) -> float: # at more configurations than would have been possible with regular cross-validation, where the number # of configurations would be determined by the number of trials divided by the number of folds (50 / 10). runhistory = smac.runhistory - print(f"Number of evaluated configurations: {len(runhistory.config_ids)}") \ No newline at end of file + print(f"Number of evaluated configurations: {len(runhistory.config_ids)}") diff --git a/setup.py b/setup.py index 85324e17d..3967dcf1f 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ def read_file(filepath: str) -> str: "scipy>=1.9.2", "psutil", "pynisher>=1.0.0", - "ConfigSpace>=0.6.1,<1.0.0", + "ConfigSpace>=1.0.0", "joblib", "scikit-learn>=1.1.2", "pyrfr>=0.9.0", diff --git a/smac/acquisition/maximizer/local_and_random_search.py b/smac/acquisition/maximizer/local_and_random_search.py index d81af2f69..71c7f86c4 100644 --- a/smac/acquisition/maximizer/local_and_random_search.py +++ b/smac/acquisition/maximizer/local_and_random_search.py @@ -145,7 +145,6 @@ def _maximize( previous_configs: list[Configuration], n_points: int, ) -> list[tuple[float, Configuration]]: - if self._uniform_configspace is not None and self._prior_sampling_fraction is not None: # Get configurations sorted by acquisition function value next_configs_by_prior_random_search_sorted = self._prior_random_search._maximize( diff --git a/smac/acquisition/maximizer/local_search.py b/smac/acquisition/maximizer/local_search.py index 3ef1ae96e..297c032a2 100644 --- a/smac/acquisition/maximizer/local_search.py +++ b/smac/acquisition/maximizer/local_search.py @@ -389,7 +389,7 @@ def _search( if acq_val[acq_index] > acq_val_candidates[i]: is_valid = False try: - neighbors[acq_index].is_valid_configuration() + neighbors[acq_index].check_valid_configuration() is_valid = True except (ValueError, ForbiddenValueError) as e: logger.debug("Local search %d: %s", i, e) diff --git a/smac/callback/metadata_callback.py b/smac/callback/metadata_callback.py index 626de5dee..95a382b5f 100644 --- a/smac/callback/metadata_callback.py +++ b/smac/callback/metadata_callback.py @@ -7,6 +7,7 @@ import smac from smac.callback.callback import Callback from smac.main.smbo import SMBO +from smac.utils.numpyencoder import NumpyEncoder __copyright__ = "Copyright 2023, AutoML.org Freiburg-Hannover" __license__ = "3-clause BSD" @@ -31,4 +32,4 @@ def on_start(self, smbo: SMBO) -> None: path.mkdir(parents=True, exist_ok=True) with open(path / "metadata.json", "w") as fp: - json.dump(meta_dict, fp, indent=2) + json.dump(meta_dict, fp, indent=2, cls=NumpyEncoder) diff --git a/smac/initial_design/abstract_initial_design.py b/smac/initial_design/abstract_initial_design.py index 7e17c88c2..d561a6772 100644 --- a/smac/initial_design/abstract_initial_design.py +++ b/smac/initial_design/abstract_initial_design.py @@ -176,7 +176,6 @@ def _transform_continuous_designs( """ params = configspace.get_hyperparameters() for idx, param in enumerate(params): - if isinstance(param, IntegerHyperparameter): design[:, idx] = param._inverse_transform(param._transform(design[:, idx])) elif isinstance(param, NumericalHyperparameter): diff --git a/smac/intensifier/abstract_intensifier.py b/smac/intensifier/abstract_intensifier.py index cb537e9cc..b7a5ae1ca 100644 --- a/smac/intensifier/abstract_intensifier.py +++ b/smac/intensifier/abstract_intensifier.py @@ -26,6 +26,7 @@ from smac.scenario import Scenario from smac.utils.configspace import get_config_hash, print_config_changes from smac.utils.logging import get_logger +from smac.utils.numpyencoder import NumpyEncoder from smac.utils.pareto_front import calculate_pareto_front, sort_by_crowding_distance __copyright__ = "Copyright 2022, automl.org" @@ -666,7 +667,7 @@ def save(self, filename: str | Path) -> None: } with open(filename, "w") as fp: - json.dump(data, fp, indent=2) + json.dump(data, fp, indent=2, cls=NumpyEncoder) def load(self, filename: str | Path) -> None: """Loads the latest state of the intensifier including the incumbents and trajectory.""" diff --git a/smac/intensifier/hyperband_budget_calculation.ipynb b/smac/intensifier/hyperband_budget_calculation.ipynb new file mode 100644 index 000000000..53766dc75 --- /dev/null +++ b/smac/intensifier/hyperband_budget_calculation.ipynb @@ -0,0 +1,73 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "14 545679.0123456805\n", + "------------------------------ HYPERBAND IN MULTI-FIDELITY ------------------------------\n", + "total budget:\t\t 10000000\n", + "total number of trials:\t 3017\n", + "number of HB rounds:\t 14.80804387568556\n", + "\n", + "\t~~~~~~~~~~~~HYPERBAND ROUND\n", + "\teta:\t\t\t\t\t 3\n", + "\tmin budget per trial:\t\t\t 1000\n", + "\tmax budget per trial:\t\t\t 100000\n", + "\ttotal number of trials per HB round:\t 206\n", + "\tbudget used per HB round:\t\t 675308.6419753085\n", + "\tnumber of brackets:\t\t\t 4\n", + "\tbudgets per stage:\t\t\t {0: [1234.567901234568, 3703.7037037037035, 11111.111111111111, 33333.33333333333, 100000.0], 1: [3703.7037037037035, 11111.111111111111, 33333.33333333333, 100000.0], 2: [11111.111111111111, 33333.33333333333, 100000.0], 3: [33333.33333333333, 100000.0], 4: [100000.0]}\n", + "\tn configs per stage:\t\t\t {0: [81, 27, 9, 3, 1], 1: [34, 11, 3, 1], 2: [15, 5, 1], 3: [8, 2], 4: [5]}\n", + "-----------------------------------------------------------------------------------------\n", + "3017\n" + ] + } + ], + "source": [ + "from smac.intensifier.hyperband_utils import get_n_trials_for_hyperband_multifidelity\n", + "\n", + "# Specify total budget\n", + "total_budget = 10_000_000\n", + "\n", + "# Optionall specify eta, min_budget and max_budget to adjust HB\n", + "min_budget = 1000\n", + "max_budget = 100000 # R\n", + "eta = 3\n", + "\n", + "\n", + "n_trials = get_n_trials_for_hyperband_multifidelity(total_budget=total_budget, eta=eta, min_budget=min_budget, max_budget=max_budget, print_summary=True)\n", + "print(n_trials)\n", + "\n", + "\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/smac/intensifier/successive_halving.py b/smac/intensifier/successive_halving.py index 01b95c6d8..58960ca67 100644 --- a/smac/intensifier/successive_halving.py +++ b/smac/intensifier/successive_halving.py @@ -384,7 +384,7 @@ def __iter__(self) -> Iterator[TrialInfo]: # noqa: D102 logger.debug("Updating tracker:") # TODO: Process stages ascending or descending? - for (bracket, stage) in list(self._tracker.keys()): + for bracket, stage in list(self._tracker.keys()): pairs = self._tracker[(bracket, stage)].copy() for seed, configs in pairs: isb_keys = self._get_instance_seed_budget_keys_by_stage(bracket=bracket, stage=stage, seed=seed) diff --git a/smac/main/smbo.py b/smac/main/smbo.py index 6138e4d64..dd6bfd354 100644 --- a/smac/main/smbo.py +++ b/smac/main/smbo.py @@ -24,6 +24,7 @@ from smac.scenario import Scenario from smac.utils.data_structures import recursively_compare_dicts from smac.utils.logging import get_logger +from smac.utils.numpyencoder import NumpyEncoder __copyright__ = "Copyright 2022, automl.org" __license__ = "3-clause BSD" @@ -414,7 +415,7 @@ def save(self) -> None: # Save optimization data with open(str(path / "optimization.json"), "w") as file: - json.dump(data, file, indent=2) + json.dump(data, file, indent=2, cls=NumpyEncoder) # And save runhistory and intensifier self._runhistory.save(path / "runhistory.json") diff --git a/smac/model/gaussian_process/kernels/rbf_kernel.py b/smac/model/gaussian_process/kernels/rbf_kernel.py index 5bf207658..13aaac49f 100644 --- a/smac/model/gaussian_process/kernels/rbf_kernel.py +++ b/smac/model/gaussian_process/kernels/rbf_kernel.py @@ -24,7 +24,6 @@ def __init__( has_conditions: bool = False, prior: AbstractPrior | None = None, ) -> None: - super().__init__( operate_on=operate_on, has_conditions=has_conditions, diff --git a/smac/model/gaussian_process/kernels/white_kernel.py b/smac/model/gaussian_process/kernels/white_kernel.py index a3fa4a61b..f8a7814a7 100644 --- a/smac/model/gaussian_process/kernels/white_kernel.py +++ b/smac/model/gaussian_process/kernels/white_kernel.py @@ -21,7 +21,6 @@ def __init__( has_conditions: bool = False, prior: AbstractPrior | None = None, ) -> None: - super().__init__( operate_on=operate_on, has_conditions=has_conditions, diff --git a/smac/model/gaussian_process/mcmc_gaussian_process.py b/smac/model/gaussian_process/mcmc_gaussian_process.py index 7c4ff4028..d6a098159 100644 --- a/smac/model/gaussian_process/mcmc_gaussian_process.py +++ b/smac/model/gaussian_process/mcmc_gaussian_process.py @@ -247,7 +247,6 @@ def _train( assert self._samples is not None for sample in self._samples: - if (sample < -50).any(): sample[sample < -50] = -50 if (sample > 50).any(): diff --git a/smac/runhistory/runhistory.py b/smac/runhistory/runhistory.py index c71384107..091e3f95b 100644 --- a/smac/runhistory/runhistory.py +++ b/smac/runhistory/runhistory.py @@ -24,6 +24,7 @@ from smac.utils.configspace import get_config_hash from smac.utils.logging import get_logger from smac.utils.multi_objective import normalize_costs +from smac.utils.numpyencoder import NumpyEncoder __copyright__ = "Copyright 2022, automl.org" __license__ = "3-clause BSD" @@ -803,6 +804,7 @@ def save(self, filename: str | Path = "runhistory.json") -> None: }, fp, indent=2, + cls=NumpyEncoder, ) def load(self, filename: str | Path, configspace: ConfigurationSpace) -> None: @@ -955,7 +957,7 @@ def _check_json_serializable( trial_value: TrialValue, ) -> None: try: - json.dumps(obj) + json.dumps(obj, cls=NumpyEncoder) except Exception as e: raise ValueError( "Cannot add %s: %s of type %s to runhistory because it raises an error during JSON encoding, " diff --git a/smac/scenario.py b/smac/scenario.py index 133ae57b0..ca0df81a2 100644 --- a/smac/scenario.py +++ b/smac/scenario.py @@ -11,9 +11,9 @@ import numpy as np from ConfigSpace import ConfigurationSpace -from ConfigSpace.read_and_write import json as cs_json from smac.utils.logging import get_logger +from smac.utils.numpyencoder import NumpyEncoder logger = get_logger(__name__) @@ -203,12 +203,11 @@ def save(self) -> None: # Save everything filename = self.output_directory / "scenario.json" with open(filename, "w") as fh: - json.dump(data, fh, indent=4) + json.dump(data, fh, indent=4, cls=NumpyEncoder) # Save configspace on its own configspace_filename = self.output_directory / "configspace.json" - with open(configspace_filename, "w") as f: - f.write(cs_json.write(self.configspace)) + self.configspace.to_json(configspace_filename) @staticmethod def load(path: Path) -> Scenario: @@ -224,9 +223,7 @@ def load(path: Path) -> Scenario: # Read configspace configspace_filename = path / "configspace.json" - with open(configspace_filename, "r") as f: - - configspace = cs_json.read(f.read()) + configspace = ConfigurationSpace.from_json(configspace_filename) data["configspace"] = configspace diff --git a/smac/utils/configspace.py b/smac/utils/configspace.py index 8f281ed3e..8224f3ef9 100644 --- a/smac/utils/configspace.py +++ b/smac/utils/configspace.py @@ -101,22 +101,22 @@ def get_types( if can_be_inactive: raise ValueError("Inactive parameters not supported for Beta and Normal Hyperparameters") - bounds[i] = (param._lower, param._upper) + bounds[i] = (param.lower_vectorized, param.upper_vectorized) elif isinstance(param, NormalIntegerHyperparameter): if can_be_inactive: raise ValueError("Inactive parameters not supported for Beta and Normal Hyperparameters") - bounds[i] = (param.nfhp._lower, param.nfhp._upper) + bounds[i] = (param.lower_vectorized, param.upper_vectorized) elif isinstance(param, BetaFloatHyperparameter): if can_be_inactive: raise ValueError("Inactive parameters not supported for Beta and Normal Hyperparameters") - bounds[i] = (param._lower, param._upper) + bounds[i] = (param.lower_vectorized, param.upper_vectorized) elif isinstance(param, BetaIntegerHyperparameter): if can_be_inactive: raise ValueError("Inactive parameters not supported for Beta and Normal Hyperparameters") - bounds[i] = (param.bfhp._lower, param.bfhp._upper) + bounds[i] = (param.lower_vectorized, param.upper_vectorized) elif not isinstance( param, ( diff --git a/smac/utils/numpyencoder.py b/smac/utils/numpyencoder.py new file mode 100644 index 000000000..c7b2b6c7b --- /dev/null +++ b/smac/utils/numpyencoder.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from typing import Any + +import json + +import numpy as np + + +class NumpyEncoder(json.JSONEncoder): + """Custom encoder for numpy data types + + From https://stackoverflow.com/a/61903895 + """ + + def default(self, obj: Any) -> Any: + """Handle numpy datatypes if present by converting to native python + + Parameters + ---------- + obj : Any + Object to serialize + + Returns + ------- + Any + Object in native python + """ + if isinstance( + obj, + ( + np.int_, + np.intc, + np.intp, + np.int8, + np.int16, + np.int32, + np.int64, + np.uint8, + np.uint16, + np.uint32, + np.uint64, + ), + ): + return int(obj) + + elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)): + return float(obj) + + elif isinstance(obj, (np.complex_, np.complex64, np.complex128)): + return {"real": obj.real, "imag": obj.imag} + + elif isinstance(obj, (np.ndarray,)): + return obj.tolist() + + elif isinstance(obj, (np.bool_)): + return bool(obj) + + elif isinstance(obj, (np.void)): + return None + + return json.JSONEncoder.default(self, obj) diff --git a/smac/utils/subspaces/__init__.py b/smac/utils/subspaces/__init__.py index 686099830..77d5b97f5 100644 --- a/smac/utils/subspaces/__init__.py +++ b/smac/utils/subspaces/__init__.py @@ -322,7 +322,7 @@ # hp_list.append(hp_new) # # We only consider plain hyperparameters -# self.cs_local.add_hyperparameters(hp_list) +# self.cs_local.add(hp_list) # forbiddens_ss = [] # forbiddens = config_space.get_forbiddens() # for forbidden in forbiddens: diff --git a/tests/conftest.py b/tests/conftest.py index 92a415162..5b26d6b70 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -143,7 +143,6 @@ def pytest_sessionfinish(session: Session, exitstatus: ExitCode) -> None: proc = psutil.Process() kill_signal = signal.SIGTERM for child in proc.children(recursive=True): - # https://stackoverflow.com/questions/57336095/access-verbosity-level-in-a-pytest-helper-function if session.config.getoption("verbose") > 0: print(child, child.cmdline()) diff --git a/tests/fixtures/configspace.py b/tests/fixtures/configspace.py index 77cf9522d..5e6e0ca3f 100644 --- a/tests/fixtures/configspace.py +++ b/tests/fixtures/configspace.py @@ -18,7 +18,7 @@ def configspace_small() -> ConfigurationSpace: c = Categorical("c", ["cat", "dog", "mouse"], default="cat") # Add all hyperparameters at once: - cs.add_hyperparameters([a, b, c]) + cs.add([a, b, c]) return cs @@ -36,7 +36,7 @@ def configspace_large() -> ConfigurationSpace: learning_rate_init = Float("learning_rate_init", (0.0001, 1.0), default=0.001, log=True) # Add all hyperparameters at once: - cs.add_hyperparameters( + cs.add( [ n_layer, n_neurons, @@ -57,6 +57,6 @@ def configspace_large() -> ConfigurationSpace: use_batch_size = InCondition(child=batch_size, parent=solver, values=["sgd", "adam"]) # We can also add multiple conditions on hyperparameters at once: - cs.add_conditions([use_lr, use_batch_size, use_lr_init]) + cs.add([use_lr, use_batch_size, use_lr_init]) return cs diff --git a/tests/fixtures/models.py b/tests/fixtures/models.py index 29f9bf152..7a7a8fcc0 100644 --- a/tests/fixtures/models.py +++ b/tests/fixtures/models.py @@ -18,7 +18,7 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x0 = Float("x0", (-5, 10), default=-3) x1 = Float("x1", (-5, 10), default=-4) - cs.add_hyperparameters([x0, x1]) + cs.add([x0, x1]) return cs @@ -58,7 +58,7 @@ def configspace(self) -> ConfigurationSpace: eta0 = Float("eta0", (0.00001, 1), default=0.1, log=True) # Add the parameters to configuration space - cs.add_hyperparameters([alpha, l1_ratio, learning_rate, eta0]) + cs.add([alpha, l1_ratio, learning_rate, eta0]) return cs diff --git a/tests/test_acquisition/test_maximizers.py b/tests/test_acquisition/test_maximizers.py index 6d373c237..d7698e0a2 100644 --- a/tests/test_acquisition/test_maximizers.py +++ b/tests/test_acquisition/test_maximizers.py @@ -54,8 +54,8 @@ def get_array(self): def configspace_branin() -> ConfigurationSpace: """Returns the branin configspace.""" cs = ConfigurationSpace() - cs.add_hyperparameter(Float("x", (-5, 10))) - cs.add_hyperparameter(Float("y", (0, 15))) + cs.add(Float("x", (-5, 10))) + cs.add(Float("y", (0, 15))) return cs @@ -195,7 +195,7 @@ def configspace() -> ConfigurationSpace: c = Float("c", (0, 1), default=0.5) # Add all hyperparameters at once: - cs.add_hyperparameters([a, b, c]) + cs.add([a, b, c]) return cs @@ -262,7 +262,6 @@ def predict_marginalized(self, X): return X, X class AcquisitionFunction: - model = Model() def __call__(self, X): @@ -333,7 +332,7 @@ def test_local_and_random_search(configspace, acquisition_function): values = rs._maximize(start_points, 100) config_origins = [] v_old = np.inf - for (v, config) in values: + for v, config in values: config_origins += [config.origin] if isinstance(v, np.ndarray): v = float(v[0]) @@ -357,7 +356,7 @@ def configspace_rosenbrock(): x2 = UniformIntegerHyperparameter("x2", -5, 5, default_value=5) x3 = CategoricalHyperparameter("x3", [5, 2, 0, 1, -1, -2, 4, -3, 3, -5, -4], default_value=5) x4 = UniformIntegerHyperparameter("x4", -5, 5, default_value=5) - uniform_cs.add_hyperparameters([x1, x2, x3, x4]) + uniform_cs.add([x1, x2, x3, x4]) return uniform_cs @@ -373,7 +372,7 @@ def configspace_prior(): "x3", [5, 2, 0, 1, -1, -2, 4, -3, 3, -5, -4], default_value=5, weights=[999, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] ) x4 = UniformIntegerHyperparameter("x4", lower=-5, upper=5, default_value=5) - prior_cs.add_hyperparameters([x1, x2, x3, x4]) + prior_cs.add([x1, x2, x3, x4]) return prior_cs diff --git a/tests/test_initial_design/test_sobol_design.py b/tests/test_initial_design/test_sobol_design.py index 8663e2972..8fbb427e8 100644 --- a/tests/test_initial_design/test_sobol_design.py +++ b/tests/test_initial_design/test_sobol_design.py @@ -24,7 +24,7 @@ def test_sobol_design(make_scenario, configspace_large): def test_max_hyperparameters(make_scenario): cs = ConfigurationSpace() hyperparameters = [Float("x%d" % (i + 1), (0, 1)) for i in range(21202)] - cs.add_hyperparameters(hyperparameters) + cs.add(hyperparameters) scenario = make_scenario(cs) diff --git a/tests/test_intensifier/test_abstract_intensifier.py b/tests/test_intensifier/test_abstract_intensifier.py index b8dc91a1c..6d878f915 100644 --- a/tests/test_intensifier/test_abstract_intensifier.py +++ b/tests/test_intensifier/test_abstract_intensifier.py @@ -110,7 +110,7 @@ def test_incumbent_selection_multi_objective(make_scenario, configspace_small, m def test_config_rejection_single_objective(configspace_small, make_scenario): - """ Tests whether configs are rejected properly if they are worse than the incumbent. """ + """Tests whether configs are rejected properly if they are worse than the incumbent.""" scenario = make_scenario(configspace_small, use_instances=False) runhistory = RunHistory() intensifier = Intensifier(scenario=scenario) @@ -118,36 +118,21 @@ def test_config_rejection_single_objective(configspace_small, make_scenario): configs = configspace_small.sample_configuration(3) - runhistory.add(config=configs[0], - cost=5, - time=0.0, - seed=0, - status=StatusType.SUCCESS, - force_update=True) + runhistory.add(config=configs[0], cost=5, time=0.0, seed=0, status=StatusType.SUCCESS, force_update=True) intensifier.update_incumbents(configs[0]) assert intensifier._rejected_config_ids == [] # add config that yielded better results, updating incumbent and sending prior incumbent to rejected - runhistory.add(config=configs[1], - cost=1, - time=0.0, - seed=0, - status=StatusType.SUCCESS, - force_update=True) + runhistory.add(config=configs[1], cost=1, time=0.0, seed=0, status=StatusType.SUCCESS, force_update=True) intensifier.update_incumbents(config=configs[1]) - + assert intensifier._rejected_config_ids == [1] # add config that is no better should thus go to rejected - runhistory.add(config=configs[2], - cost=1, - time=0.0, - seed=0, - status=StatusType.SUCCESS, - force_update=True) + runhistory.add(config=configs[2], cost=1, time=0.0, seed=0, status=StatusType.SUCCESS, force_update=True) intensifier.update_incumbents(config=configs[2]) - + assert intensifier._rejected_config_ids == [1, 3] diff --git a/tests/test_model/_test_gp_gpytorch.py b/tests/test_model/_test_gp_gpytorch.py index 95f3a59d8..d9e90f9b3 100644 --- a/tests/test_model/_test_gp_gpytorch.py +++ b/tests/test_model/_test_gp_gpytorch.py @@ -325,7 +325,6 @@ def test_sampling_shape(self): X = np.arange(-5, 5, 0.1).reshape((-1, 1)) X_test = np.arange(-5.05, 5.05, 0.1).reshape((-1, 1)) for shape in (None, (-1, 1)): - if shape is None: y = np.sin(X).flatten() else: diff --git a/tests/test_model/test_gp.py b/tests/test_model/test_gp.py index 0c72eb2fa..9e21925bc 100644 --- a/tests/test_model/test_gp.py +++ b/tests/test_model/test_gp.py @@ -240,7 +240,6 @@ def __call__(self, X, eval_gradient=True, clone_kernel=True): raise np.linalg.LinAlgError with patch.object(sklearn.gaussian_process.GaussianProcessRegressor, "log_marginal_likelihood", Dummy().__call__): - seed = 1 rs = np.random.RandomState(seed) X, Y, n_dims = get_cont_data(rs) @@ -265,7 +264,6 @@ def __call__(self, X, Y=None): dummy = Dummy() with patch.object(GaussianProcess, "predict", dummy.__call__): - seed = 1 rs = np.random.RandomState(seed) @@ -375,7 +373,6 @@ def test_sampling_shape(): X = np.arange(-5, 5, 0.1).reshape((-1, 1)) X_test = np.arange(-5.05, 5.05, 0.1).reshape((-1, 1)) for shape in (None, (-1, 1)): - if shape is None: y = np.sin(X).flatten() else: diff --git a/tests/test_model/test_rf.py b/tests/test_model/test_rf.py index e59b2ded1..c81549a16 100644 --- a/tests/test_model/test_rf.py +++ b/tests/test_model/test_rf.py @@ -127,7 +127,6 @@ def test_predict_marginalized(): def test_predict_marginalized_mocked(): - rs = np.random.RandomState(1) F = {} for i in range(10): diff --git a/tests/test_runhistory/test_runhistory.py b/tests/test_runhistory/test_runhistory.py index a1b7616ca..428ec54ad 100644 --- a/tests/test_runhistory/test_runhistory.py +++ b/tests/test_runhistory/test_runhistory.py @@ -75,7 +75,6 @@ def test_add_and_pickle(runhistory, config1): def test_illegal_input(runhistory): - with pytest.raises(TypeError, match="Configuration must not be None."): runhistory.add(config=None, cost=1.23, time=2.34, status=StatusType.SUCCESS) @@ -87,7 +86,6 @@ def test_illegal_input(runhistory): def test_add_multiple_times(runhistory, config1): - for i in range(5): runhistory.add( config=config1, @@ -294,7 +292,6 @@ def test_full_update2(runhistory, config1, config2): def test_incremental_update(runhistory, config1): - runhistory.add( config=config1, cost=10, @@ -319,7 +316,6 @@ def test_incremental_update(runhistory, config1): def test_multiple_budgets(runhistory, config1): - runhistory.add( config=config1, cost=10, @@ -382,7 +378,6 @@ def test_get_configs_per_budget(runhistory, config1, config2, config3): def test_json_origin(configspace_small, config1): - for i, origin in enumerate(["test_origin", None]): config1.origin = origin runhistory = RunHistory() diff --git a/tests/test_runhistory/test_runhistory_encoder.py b/tests/test_runhistory/test_runhistory_encoder.py index e3a8f5730..d9aaf0769 100644 --- a/tests/test_runhistory/test_runhistory_encoder.py +++ b/tests/test_runhistory/test_runhistory_encoder.py @@ -1,5 +1,7 @@ import numpy as np import pytest +from ConfigSpace import Configuration +from ConfigSpace.hyperparameters import CategoricalHyperparameter from smac.multi_objective.aggregation_strategy import MeanAggregationStrategy from smac.runhistory.encoder import ( @@ -13,9 +15,6 @@ from smac.runhistory.encoder.encoder import RunHistoryEncoder from smac.runner.abstract_runner import StatusType -from ConfigSpace import Configuration -from ConfigSpace.hyperparameters import CategoricalHyperparameter - @pytest.fixture def configs(configspace_small): @@ -42,9 +41,7 @@ def test_transform(runhistory, make_scenario, configspace_small, configs): ) # Normal encoder - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory # TODO: Please replace with the more general solution once ConfigSpace 1.0 @@ -54,9 +51,7 @@ def test_transform(runhistory, make_scenario, configspace_small, configs): # Categoricals are upperbounded by their size, rest of hyperparameters are # upperbounded by 1. upper_bounds = { - hp.name: (hp.get_size() - 1) - if isinstance(hp, CategoricalHyperparameter) - else 1.0 + hp.name: (hp.get_size() - 1) if isinstance(hp, CategoricalHyperparameter) else 1.0 for hp in configspace_small.get_hyperparameters() } # Need to ensure they match the order in the Configuration vectorized form @@ -73,49 +68,37 @@ def test_transform(runhistory, make_scenario, configspace_small, configs): assert ((X1 <= upper) & (X1 >= lower)).all() # Log encoder - encoder = RunHistoryLogEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryLogEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryLogScaledEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryLogScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryScaledEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryInverseScaledEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryInverseScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistorySqrtScaledEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistorySqrtScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryEIPSEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEIPSEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() @@ -160,9 +143,7 @@ def test_transform_conditionals(runhistory, make_scenario, configspace_large): status=StatusType.SUCCESS, ) - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() @@ -184,9 +165,7 @@ def test_multi_objective(runhistory, make_scenario, configspace_small, configs): # Multi objective algorithm must be set with pytest.raises(AssertionError): - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory _, Y = encoder.transform() @@ -242,9 +221,7 @@ def test_ignore(runhistory, make_scenario, configspace_small, configs): ) # Normal encoder - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X1, Y1 = encoder.transform() @@ -283,14 +260,10 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs): budget=2, ) - runhistory.add( - config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2 - ) + runhistory.add(config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2) # Normal encoder - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform(budget_subset=[2]) assert Y.tolist() == [[99999999]] @@ -319,14 +292,10 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs): budget=2, ) - runhistory.add( - config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2 - ) + runhistory.add(config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2) # Normal encoder - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform(budget_subset=[2]) assert Y.tolist() == [[99999999]] @@ -338,20 +307,12 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs): def test_lower_budget_states(runhistory, make_scenario, configspace_small, configs): """Tests lower budgets based on budget subset and considered states.""" scenario = make_scenario(configspace_small) - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory - runhistory.add( - config=configs[0], cost=1, time=1, status=StatusType.SUCCESS, budget=3 - ) - runhistory.add( - config=configs[0], cost=2, time=2, status=StatusType.SUCCESS, budget=4 - ) - runhistory.add( - config=configs[0], cost=3, time=4, status=StatusType.TIMEOUT, budget=5 - ) + runhistory.add(config=configs[0], cost=1, time=1, status=StatusType.SUCCESS, budget=3) + runhistory.add(config=configs[0], cost=2, time=2, status=StatusType.SUCCESS, budget=4) + runhistory.add(config=configs[0], cost=3, time=4, status=StatusType.TIMEOUT, budget=5) # We request a higher budget but can't find it, so we expect an empty list X, Y = encoder.transform(budget_subset=[500]) From 54cc71a5921054eca3f38af9ec3b7548f1731624 Mon Sep 17 00:00:00 2001 From: benjamc Date: Tue, 23 Jul 2024 19:01:28 +0200 Subject: [PATCH 43/57] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 045253993..72bdf2922 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# 2.1.1 +# 2.2.0 ## Features - Add example to specify total budget (fidelity units) instead of n_trials for multi-fidelity/Hyperband (#1121) From 3a7e4979d431401e1a7a6e172758972c112381c5 Mon Sep 17 00:00:00 2001 From: benjamc Date: Tue, 23 Jul 2024 19:03:20 +0200 Subject: [PATCH 44/57] Update version --- CITATION.cff | 2 +- Makefile | 2 +- docs/conf.py | 3 ++- smac/__init__.py | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index ece8325c6..2279146d7 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,7 +9,7 @@ date-released: "2016-08-17" url: "https://automl.github.io/SMAC3/master/index.html" repository-code: "https://github.com/automl/SMAC3" -version: "2.1.0" +version: "2.2.0" type: "software" keywords: diff --git a/Makefile b/Makefile index 195773082..145d1bb14 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ SHELL := /bin/bash NAME := SMAC3 PACKAGE_NAME := smac -VERSION := 2.1.0 +VERSION := 2.2.0 DIR := "${CURDIR}" SOURCE_DIR := ${PACKAGE_NAME} diff --git a/docs/conf.py b/docs/conf.py index afb4d2605..4adbf4cd2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,7 +12,8 @@ "version": version, "versions": { f"v{version}": "#", - "v2.1.0": "https://automl.github.io/SMAC3/v2.2.0/", + "v2.2.0": "https://automl.github.io/SMAC3/v2.2.0/", + "v2.1.0": "https://automl.github.io/SMAC3/v2.1.0/", "v2.0.1": "https://automl.github.io/SMAC3/v2.0.1/", "v2.0.0": "https://automl.github.io/SMAC3/v2.0.0/", "v2.0.0b1": "https://automl.github.io/SMAC3/v2.0.0b1/", diff --git a/smac/__init__.py b/smac/__init__.py index 45d7df2d2..439dc48fa 100644 --- a/smac/__init__.py +++ b/smac/__init__.py @@ -19,7 +19,7 @@ Copyright {datetime.date.today().strftime('%Y')}, Marius Lindauer, Katharina Eggensperger, Matthias Feurer, André Biedenkapp, Difan Deng, Carolin Benjamins, Tim Ruhkopf, René Sass and Frank Hutter""" -version = "2.1.0" +version = "2.2.0" try: From 4a343fba17808175eb348c4cafc26d419e8496c5 Mon Sep 17 00:00:00 2001 From: benjamc Date: Tue, 23 Jul 2024 19:09:08 +0200 Subject: [PATCH 45/57] Adapt to new ConfigSpace API --- tests/test_runhistory/test_runhistory_encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_runhistory/test_runhistory_encoder.py b/tests/test_runhistory/test_runhistory_encoder.py index 9861281f5..ba542c9a1 100644 --- a/tests/test_runhistory/test_runhistory_encoder.py +++ b/tests/test_runhistory/test_runhistory_encoder.py @@ -56,7 +56,7 @@ def test_transform(runhistory, make_scenario, configspace_small, configs): # Categoricals are upperbounded by their size, rest of hyperparameters are # upperbounded by 1. upper_bounds = { - hp.name: (hp.get_size() - 1) if isinstance(hp, CategoricalHyperparameter) else 1.0 + hp.name: (hp.size - 1) if isinstance(hp, CategoricalHyperparameter) else 1.0 for hp in configspace_small.get_hyperparameters() } # Need to ensure they match the order in the Configuration vectorized form From 0550b7c26257fbf36695d83c229d231c4e084f28 Mon Sep 17 00:00:00 2001 From: benjamc Date: Tue, 23 Jul 2024 19:12:30 +0200 Subject: [PATCH 46/57] Fix merge errors --- smac/acquisition/maximizer/local_and_random_search.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/smac/acquisition/maximizer/local_and_random_search.py b/smac/acquisition/maximizer/local_and_random_search.py index 8a706cda6..258b71453 100644 --- a/smac/acquisition/maximizer/local_and_random_search.py +++ b/smac/acquisition/maximizer/local_and_random_search.py @@ -69,8 +69,6 @@ def __init__( seed: int = 0, uniform_configspace: ConfigurationSpace | None = None, prior_sampling_fraction: float | None = None, - uniform_configspace: ConfigurationSpace | None = None, - prior_sampling_fraction: float | None = None, ) -> None: super().__init__( configspace, @@ -237,10 +235,6 @@ def _maximize( additional_start_points=random_starting_points, ) - next_configs_by_acq_value = next_configs_by_local_search - additional_start_points=random_starting_points, - ) - next_configs_by_acq_value = next_configs_by_local_search next_configs_by_acq_value.sort(reverse=True, key=lambda x: x[0]) first_five = [f"{_[0]} ({_[1].origin})" for _ in next_configs_by_acq_value[:5]] From 46d3863da892236137a6d69a32f7969eba574ea9 Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 24 Jul 2024 10:03:28 +0200 Subject: [PATCH 47/57] refactor: remove notebook --- .../hyperband_budget_calculation.ipynb | 73 ------------------- 1 file changed, 73 deletions(-) delete mode 100644 smac/intensifier/hyperband_budget_calculation.ipynb diff --git a/smac/intensifier/hyperband_budget_calculation.ipynb b/smac/intensifier/hyperband_budget_calculation.ipynb deleted file mode 100644 index 53766dc75..000000000 --- a/smac/intensifier/hyperband_budget_calculation.ipynb +++ /dev/null @@ -1,73 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "14 545679.0123456805\n", - "------------------------------ HYPERBAND IN MULTI-FIDELITY ------------------------------\n", - "total budget:\t\t 10000000\n", - "total number of trials:\t 3017\n", - "number of HB rounds:\t 14.80804387568556\n", - "\n", - "\t~~~~~~~~~~~~HYPERBAND ROUND\n", - "\teta:\t\t\t\t\t 3\n", - "\tmin budget per trial:\t\t\t 1000\n", - "\tmax budget per trial:\t\t\t 100000\n", - "\ttotal number of trials per HB round:\t 206\n", - "\tbudget used per HB round:\t\t 675308.6419753085\n", - "\tnumber of brackets:\t\t\t 4\n", - "\tbudgets per stage:\t\t\t {0: [1234.567901234568, 3703.7037037037035, 11111.111111111111, 33333.33333333333, 100000.0], 1: [3703.7037037037035, 11111.111111111111, 33333.33333333333, 100000.0], 2: [11111.111111111111, 33333.33333333333, 100000.0], 3: [33333.33333333333, 100000.0], 4: [100000.0]}\n", - "\tn configs per stage:\t\t\t {0: [81, 27, 9, 3, 1], 1: [34, 11, 3, 1], 2: [15, 5, 1], 3: [8, 2], 4: [5]}\n", - "-----------------------------------------------------------------------------------------\n", - "3017\n" - ] - } - ], - "source": [ - "from smac.intensifier.hyperband_utils import get_n_trials_for_hyperband_multifidelity\n", - "\n", - "# Specify total budget\n", - "total_budget = 10_000_000\n", - "\n", - "# Optionall specify eta, min_budget and max_budget to adjust HB\n", - "min_budget = 1000\n", - "max_budget = 100000 # R\n", - "eta = 3\n", - "\n", - "\n", - "n_trials = get_n_trials_for_hyperband_multifidelity(total_budget=total_budget, eta=eta, min_budget=min_budget, max_budget=max_budget, print_summary=True)\n", - "print(n_trials)\n", - "\n", - "\n", - "\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 89089febe07eac68763c5106e1bfb10f5287dd44 Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 24 Jul 2024 10:07:21 +0200 Subject: [PATCH 48/57] Fix merge errors --- .../maximizer/local_and_random_search.py | 55 ------------------- 1 file changed, 55 deletions(-) diff --git a/smac/acquisition/maximizer/local_and_random_search.py b/smac/acquisition/maximizer/local_and_random_search.py index 258b71453..71c7f86c4 100644 --- a/smac/acquisition/maximizer/local_and_random_search.py +++ b/smac/acquisition/maximizer/local_and_random_search.py @@ -21,7 +21,6 @@ class LocalAndSortedRandomSearch(AbstractAcquisitionMaximizer): """Implement SMAC's default acquisition function optimization. - This optimizer performs local search from the previous best points according to the acquisition This optimizer performs local search from the previous best points according to the acquisition function, uses the acquisition function to sort randomly sampled configurations. Random configurations are interleaved by the main SMAC code. @@ -32,10 +31,6 @@ class LocalAndSortedRandomSearch(AbstractAcquisitionMaximizer): Parameters ---------- configspace : ConfigurationSpace - uniform_configspace : ConfigurationSpace - A version of the user-defined ConfigurationSpace where all parameters are uniform (or have their weights removed - in the case of a categorical hyperparameter). Can optionally be given and sampling ratios be defined via the - `prior_sampling_fraction` parameter. uniform_configspace : ConfigurationSpace A version of the user-defined ConfigurationSpace where all parameters are uniform (or have their weights removed in the case of a categorical hyperparameter). Can optionally be given and sampling ratios be defined via the @@ -52,9 +47,6 @@ class LocalAndSortedRandomSearch(AbstractAcquisitionMaximizer): prior_sampling_fraction: float, defaults to 0.5 The ratio of random samples that are taken from the user-defined ConfigurationSpace, as opposed to the uniform version (needs `uniform_configspace`to be defined). - prior_sampling_fraction: float, defaults to 0.5 - The ratio of random samples that are taken from the user-defined ConfigurationSpace, as opposed to the uniform - version (needs `uniform_configspace`to be defined). seed : int, defaults to 0 """ @@ -77,28 +69,6 @@ def __init__( seed=seed, ) - if uniform_configspace is not None and prior_sampling_fraction is None: - prior_sampling_fraction = 0.5 - if uniform_configspace is None and prior_sampling_fraction is not None: - raise ValueError("If `prior_sampling_fraction` is given, `uniform_configspace` must be defined.") - if uniform_configspace is not None and prior_sampling_fraction is not None: - self._prior_random_search = RandomSearch( - acquisition_function=acquisition_function, - configspace=configspace, - seed=seed, - ) - - self._uniform_random_search = RandomSearch( - acquisition_function=acquisition_function, - configspace=uniform_configspace, - seed=seed, - ) - else: - self._random_search = RandomSearch( - configspace=configspace, - acquisition_function=acquisition_function, - seed=seed, - ) if uniform_configspace is not None and prior_sampling_fraction is None: prior_sampling_fraction = 0.5 if uniform_configspace is None and prior_sampling_fraction is not None: @@ -133,8 +103,6 @@ def __init__( self._local_search_iterations = local_search_iterations self._prior_sampling_fraction = prior_sampling_fraction self._uniform_configspace = uniform_configspace - self._prior_sampling_fraction = prior_sampling_fraction - self._uniform_configspace = uniform_configspace @property def acquisition_function(self) -> AbstractAcquisitionFunction | None: # noqa: D102 @@ -144,11 +112,6 @@ def acquisition_function(self) -> AbstractAcquisitionFunction | None: # noqa: D @acquisition_function.setter def acquisition_function(self, acquisition_function: AbstractAcquisitionFunction) -> None: self._acquisition_function = acquisition_function - if self._uniform_configspace is not None: - self._prior_random_search._acquisition_function = acquisition_function - self._uniform_random_search._acquisition_function = acquisition_function - else: - self._random_search._acquisition_function = acquisition_function if self._uniform_configspace is not None: self._prior_random_search._acquisition_function = acquisition_function self._uniform_random_search._acquisition_function = acquisition_function @@ -174,21 +137,6 @@ def meta(self) -> dict[str, Any]: # noqa: D102 "local_search": self._local_search.meta, } ) - if self._uniform_configspace is None: - meta.update( - { - "random_search": self._random_search.meta, - "local_search": self._local_search.meta, - } - ) - else: - meta.update( - { - "prior_random_search": self._prior_random_search.meta, - "uniform_random_search": self._uniform_random_search.meta, - "local_search": self._local_search.meta, - } - ) return meta @@ -223,9 +171,6 @@ def _maximize( _sorted=True, ) - # Choose the best self._local_search_iterations random configs to start the local search, and choose only - # incumbent from previous configs - random_starting_points = next_configs_by_random_search_sorted[: self._local_search_iterations] # Choose the best self._local_search_iterations random configs to start the local search, and choose only # incumbent from previous configs random_starting_points = next_configs_by_random_search_sorted[: self._local_search_iterations] From eb3b6526ec5c7c7ee1b626ecd215e9accb3847f7 Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 24 Jul 2024 10:08:54 +0200 Subject: [PATCH 49/57] Make format --- .../1_basics/7_parallelization_cluster.py | 2 +- .../test_runhistory_encoder.py | 75 +++++-------------- 2 files changed, 19 insertions(+), 58 deletions(-) diff --git a/examples/1_basics/7_parallelization_cluster.py b/examples/1_basics/7_parallelization_cluster.py index e5c33f94d..2467f7d22 100644 --- a/examples/1_basics/7_parallelization_cluster.py +++ b/examples/1_basics/7_parallelization_cluster.py @@ -103,7 +103,7 @@ def train(self, config: Configuration, seed: int = 0) -> float: # if you would like to limit the resources consumption of each function evaluation with pynisher, you need to # set nanny as False # Otherwise, an error `daemonic processes are not allowed to have children` will raise! - nanny=False # if you do not use pynisher to limit the memory/time usage, feel free to set this one as True + nanny=False, # if you do not use pynisher to limit the memory/time usage, feel free to set this one as True ) cluster.scale(jobs=n_workers) diff --git a/tests/test_runhistory/test_runhistory_encoder.py b/tests/test_runhistory/test_runhistory_encoder.py index ba542c9a1..ac9824534 100644 --- a/tests/test_runhistory/test_runhistory_encoder.py +++ b/tests/test_runhistory/test_runhistory_encoder.py @@ -15,9 +15,6 @@ from smac.runhistory.encoder.encoder import RunHistoryEncoder from smac.runner.abstract_runner import StatusType -from ConfigSpace import Configuration -from ConfigSpace.hyperparameters import CategoricalHyperparameter - @pytest.fixture def configs(configspace_small): @@ -44,9 +41,7 @@ def test_transform(runhistory, make_scenario, configspace_small, configs): ) # Normal encoder - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory # TODO: Please replace with the more general solution once ConfigSpace 1.0 @@ -73,49 +68,37 @@ def test_transform(runhistory, make_scenario, configspace_small, configs): assert ((X1 <= upper) & (X1 >= lower)).all() # Log encoder - encoder = RunHistoryLogEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryLogEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryLogScaledEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryLogScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryScaledEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryInverseScaledEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryInverseScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistorySqrtScaledEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistorySqrtScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() assert ((X <= upper) & (X >= lower)).all() - encoder = RunHistoryEIPSEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEIPSEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() @@ -160,9 +143,7 @@ def test_transform_conditionals(runhistory, make_scenario, configspace_large): status=StatusType.SUCCESS, ) - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() @@ -184,9 +165,7 @@ def test_multi_objective(runhistory, make_scenario, configspace_small, configs): # Multi objective algorithm must be set with pytest.raises(AssertionError): - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory _, Y = encoder.transform() @@ -242,9 +221,7 @@ def test_ignore(runhistory, make_scenario, configspace_small, configs): ) # Normal encoder - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X1, Y1 = encoder.transform() @@ -283,14 +260,10 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs): budget=2, ) - runhistory.add( - config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2 - ) + runhistory.add(config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2) # Normal encoder - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform(budget_subset=[2]) assert Y.tolist() == [[99999999]] @@ -319,14 +292,10 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs): budget=2, ) - runhistory.add( - config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2 - ) + runhistory.add(config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2) # Normal encoder - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform(budget_subset=[2]) assert Y.tolist() == [[99999999]] @@ -338,20 +307,12 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs): def test_lower_budget_states(runhistory, make_scenario, configspace_small, configs): """Tests lower budgets based on budget subset and considered states.""" scenario = make_scenario(configspace_small) - encoder = RunHistoryEncoder( - scenario=scenario, considered_states=[StatusType.SUCCESS] - ) + encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory - runhistory.add( - config=configs[0], cost=1, time=1, status=StatusType.SUCCESS, budget=3 - ) - runhistory.add( - config=configs[0], cost=2, time=2, status=StatusType.SUCCESS, budget=4 - ) - runhistory.add( - config=configs[0], cost=3, time=4, status=StatusType.TIMEOUT, budget=5 - ) + runhistory.add(config=configs[0], cost=1, time=1, status=StatusType.SUCCESS, budget=3) + runhistory.add(config=configs[0], cost=2, time=2, status=StatusType.SUCCESS, budget=4) + runhistory.add(config=configs[0], cost=3, time=4, status=StatusType.TIMEOUT, budget=5) # We request a higher budget but can't find it, so we expect an empty list X, Y = encoder.transform(budget_subset=[500]) From f0c3c99983ef1ac99d4ca42fae708a03c25c24b7 Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 24 Jul 2024 10:17:00 +0200 Subject: [PATCH 50/57] Add test case for numpy encoder --- tests/test_utils/test_numpy_encoder.py | 39 ++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tests/test_utils/test_numpy_encoder.py diff --git a/tests/test_utils/test_numpy_encoder.py b/tests/test_utils/test_numpy_encoder.py new file mode 100644 index 000000000..7b9d0f20a --- /dev/null +++ b/tests/test_utils/test_numpy_encoder.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import json +import numpy as np +import pytest +from smac.utils.numpyencoder import NumpyEncoder + +# Test cases for NumpyEncoder +def test_numpy_encoder(): + data = { + "int": np.int32(1), + "float": np.float32(1.23), + "complex": np.complex64(1 + 2j), + "array": np.array([1, 2, 3]), + "bool": np.bool_(True), + "void": np.void(b'void') + } + + expected_output = { + "int": 1, + "float": 1.23, + "complex": {"real": 1.0, "imag": 2.0}, + "array": [1, 2, 3], + "bool": True, + "void": None + } + + encoded_data = json.dumps(data, cls=NumpyEncoder) + decoded_data = json.loads(encoded_data) + + assert np.isclose(decoded_data["float"], expected_output["float"]) # float ist not exactly the same + del decoded_data["float"] + del expected_output["float"] + assert decoded_data == expected_output + +# Test if default method raises TypeError for unsupported types +def test_numpy_encoder_unsupported_type(): + with pytest.raises(TypeError): + json.dumps(set([1, 2, 3]), cls=NumpyEncoder) \ No newline at end of file From ef9315137b7ee8e3a846f225ef367010dff1eee7 Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 24 Jul 2024 11:54:12 +0200 Subject: [PATCH 51/57] Make format --- tests/test_utils/test_numpy_encoder.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_utils/test_numpy_encoder.py b/tests/test_utils/test_numpy_encoder.py index 7b9d0f20a..9074e3cd8 100644 --- a/tests/test_utils/test_numpy_encoder.py +++ b/tests/test_utils/test_numpy_encoder.py @@ -1,10 +1,13 @@ from __future__ import annotations import json + import numpy as np import pytest + from smac.utils.numpyencoder import NumpyEncoder + # Test cases for NumpyEncoder def test_numpy_encoder(): data = { @@ -13,16 +16,16 @@ def test_numpy_encoder(): "complex": np.complex64(1 + 2j), "array": np.array([1, 2, 3]), "bool": np.bool_(True), - "void": np.void(b'void') + "void": np.void(b"void"), } - + expected_output = { "int": 1, "float": 1.23, "complex": {"real": 1.0, "imag": 2.0}, "array": [1, 2, 3], "bool": True, - "void": None + "void": None, } encoded_data = json.dumps(data, cls=NumpyEncoder) @@ -33,7 +36,8 @@ def test_numpy_encoder(): del expected_output["float"] assert decoded_data == expected_output + # Test if default method raises TypeError for unsupported types def test_numpy_encoder_unsupported_type(): with pytest.raises(TypeError): - json.dumps(set([1, 2, 3]), cls=NumpyEncoder) \ No newline at end of file + json.dumps(set([1, 2, 3]), cls=NumpyEncoder) From 5409bc0131882d224002ed69e7d668483fb48418 Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 24 Jul 2024 11:54:22 +0200 Subject: [PATCH 52/57] Fix calculation --- smac/intensifier/hyperband_utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/smac/intensifier/hyperband_utils.py b/smac/intensifier/hyperband_utils.py index ad8773a71..2171a5903 100644 --- a/smac/intensifier/hyperband_utils.py +++ b/smac/intensifier/hyperband_utils.py @@ -52,7 +52,7 @@ def determine_HB(min_budget: float, max_budget: float, eta: int = 3) -> dict: "budgets_in_stage": _budgets_in_stage, "trials_used": total_trials, "budget_used": total_budget, - "number_of_brackets": _s_max, + "number_of_brackets": len(_max_iterations), } @@ -75,13 +75,19 @@ def determine_hyperband_for_multifidelity( Returns ------- dict - Info about the Hyperband round + Info about one Hyperband round "max_iterations" "n_configs_in_stage" "budgets_in_stage" "trials_used" "budget_used" "number_of_brackets" + Info about whole optimization + "n_trials" + "total_budget" + "eta" + "min_budget" + "max_budget" """ # Determine the HB @@ -109,9 +115,6 @@ def determine_hyperband_for_multifidelity( # budget left. We can not add full c but the number of trials that still fit remaining_budget = max(0, remaining_budget - b * c) - # print(stage, b, c) - # print("-"*20, remaining_trials, remaining_budget) - n_trials = int(number_of_full_hb_rounds * trials_used_per_hb_round + remaining_trials) hyperband_info = hyperband_round From 29264c750d0cbda07b85eab3b93d019df9dd5298 Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 24 Jul 2024 11:54:35 +0200 Subject: [PATCH 53/57] Add test for Hyperband calculation --- .../test_intensifier/test_hyperband_utils.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tests/test_intensifier/test_hyperband_utils.py diff --git a/tests/test_intensifier/test_hyperband_utils.py b/tests/test_intensifier/test_hyperband_utils.py new file mode 100644 index 000000000..49a697b5e --- /dev/null +++ b/tests/test_intensifier/test_hyperband_utils.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from smac.intensifier.hyperband_utils import ( + determine_HB, + determine_hyperband_for_multifidelity, + get_n_trials_for_hyperband_multifidelity, +) + + +def test_determine_HB(): + min_budget = 1.0 + max_budget = 81.0 + eta = 3 + + result = determine_HB(min_budget=min_budget, max_budget=max_budget, eta=eta) + + # Test table 1 from https://arxiv.org/pdf/1603.06560.pdf + expected_max_iterations = {0: 5, 1: 4, 2: 3, 3: 2, 4: 1} + expected_n_configs_in_stage = { + 0: [81, 27, 9, 3, 1], + 1: [34, 11, 3, 1], + 2: [15, 5, 1], + 3: [8, 2], + 4: [5], + } + expected_budgets_in_stage = { + 0: [1, 3, 9, 27, 81], + 1: [3, 9, 27, 81], + 2: [9, 27, 81], + 3: [27, 81], + 4: [81], + } + expected_trials_used = 206 + expected_budget_used = 547 + expected_number_of_brackets = 5 + + assert result["max_iterations"] == expected_max_iterations + assert result["n_configs_in_stage"] == expected_n_configs_in_stage + assert result["budgets_in_stage"] == expected_budgets_in_stage + assert result["trials_used"] == expected_trials_used + assert result["budget_used"] == expected_budget_used + assert result["number_of_brackets"] == expected_number_of_brackets + + +def test_determine_hyperband_for_multifidelity(): + total_budget = 1000.0 + min_budget = 1.0 + max_budget = 81.0 + eta = 3 + + result = determine_hyperband_for_multifidelity( + total_budget=total_budget, min_budget=min_budget, max_budget=max_budget, eta=eta + ) + + expected_n_trials = 206 + 137 # 206 trials for one full round, and additional trials for the remaining budget + + assert result["n_trials"] == expected_n_trials + assert result["total_budget"] == total_budget + assert result["eta"] == eta + assert result["min_budget"] == min_budget + assert result["max_budget"] == max_budget + + +def test_get_n_trials_for_hyperband_multifidelity(): + total_budget = 1000.0 + min_budget = 1.0 + max_budget = 81.0 + eta = 3 + + n_trials = get_n_trials_for_hyperband_multifidelity( + total_budget=total_budget, min_budget=min_budget, max_budget=max_budget, eta=eta + ) + + assert n_trials == (206 + 137) From f9f8df2728de9c2f7ced55362384542627788d22 Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 24 Jul 2024 14:50:39 +0200 Subject: [PATCH 54/57] Update comment --- tests/test_intensifier/test_hyperband_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_intensifier/test_hyperband_utils.py b/tests/test_intensifier/test_hyperband_utils.py index 49a697b5e..33179d0ad 100644 --- a/tests/test_intensifier/test_hyperband_utils.py +++ b/tests/test_intensifier/test_hyperband_utils.py @@ -14,7 +14,7 @@ def test_determine_HB(): result = determine_HB(min_budget=min_budget, max_budget=max_budget, eta=eta) - # Test table 1 from https://arxiv.org/pdf/1603.06560.pdf + # Follow algorithm (not the table!) from https://arxiv.org/pdf/1603.06560.pdf (see https://github.com/automl/SMAC3/issues/977) expected_max_iterations = {0: 5, 1: 4, 2: 3, 3: 2, 4: 1} expected_n_configs_in_stage = { 0: [81, 27, 9, 3, 1], From 59870f5d4a3606f25c7ac5e350e9adb0b4cb0401 Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 24 Jul 2024 14:52:10 +0200 Subject: [PATCH 55/57] Update docstring --- smac/intensifier/hyperband_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/smac/intensifier/hyperband_utils.py b/smac/intensifier/hyperband_utils.py index 2171a5903..71261dee5 100644 --- a/smac/intensifier/hyperband_utils.py +++ b/smac/intensifier/hyperband_utils.py @@ -156,14 +156,14 @@ def print_hyperband_summary(hyperband_info: dict) -> None: def get_n_trials_for_hyperband_multifidelity( total_budget: float, min_budget: float, max_budget: float, eta: int = 3, print_summary: bool = True ) -> int: - """Caculate the number of trials needed for multi-fidelity optimization + """Calculate the number of trials needed for multi-fidelity optimization Specify the total budget and find out how many trials that equals. Parameters ---------- total_budget : float - Total budget for the complete optimization in fidelity units + Total budget for the complete optimization in fidelity units. A fidelity unit can be one epoch or a fraction of a dataset size. min_budget : float Minimum budget per trial in fidelity units max_budget : float From abfe86a8ccba769f17a6945ae0c0d3dfb1d96028 Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 24 Jul 2024 16:17:04 +0200 Subject: [PATCH 56/57] Fix pre-commit --- smac/intensifier/hyperband_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/smac/intensifier/hyperband_utils.py b/smac/intensifier/hyperband_utils.py index 71261dee5..581325b95 100644 --- a/smac/intensifier/hyperband_utils.py +++ b/smac/intensifier/hyperband_utils.py @@ -163,7 +163,8 @@ def get_n_trials_for_hyperband_multifidelity( Parameters ---------- total_budget : float - Total budget for the complete optimization in fidelity units. A fidelity unit can be one epoch or a fraction of a dataset size. + Total budget for the complete optimization in fidelity units. + A fidelity unit can be one epoch or a fraction of a dataset size. min_budget : float Minimum budget per trial in fidelity units max_budget : float From 7058b456666451dcb19968610f04bc8953432ec9 Mon Sep 17 00:00:00 2001 From: benjamc Date: Wed, 24 Jul 2024 16:18:15 +0200 Subject: [PATCH 57/57] Fix pre-commit --- smac/intensifier/hyperband_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smac/intensifier/hyperband_utils.py b/smac/intensifier/hyperband_utils.py index 581325b95..77f6a748c 100644 --- a/smac/intensifier/hyperband_utils.py +++ b/smac/intensifier/hyperband_utils.py @@ -163,7 +163,7 @@ def get_n_trials_for_hyperband_multifidelity( Parameters ---------- total_budget : float - Total budget for the complete optimization in fidelity units. + Total budget for the complete optimization in fidelity units. A fidelity unit can be one epoch or a fraction of a dataset size. min_budget : float Minimum budget per trial in fidelity units