diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000000..978ac45770 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +github: automl diff --git a/.github/stale.yml b/.github/stale.yml deleted file mode 100644 index 5f6a3d7b54..0000000000 --- a/.github/stale.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Number of days of inactivity before an issue becomes stale -daysUntilStale: 60 - -# Number of days of inactivity before a stale issue is closed -daysUntilClose: 14 - -# Issues with these labels will never be considered stale -exemptLabels: - - bug - - dependency - - documentation - - enhancement - - feature - - test - - example - - discussion - - duplicate - - question - -# Label to use when marking an issue as stale -staleLabel: stale - -# Comment to post when marking an issue as stale. Set to `false` to disable -markComment: > - This issue has been automatically marked as stale because it has not had - recent activity. It will be closed if no further activity occurs. Thank you - for your contributions. - -# Comment to post when closing a stale issue. Set to `false` to disable -closeComment: false \ No newline at end of file diff --git a/.github/workflows/citation.yml b/.github/workflows/citation.yml index 9ca4ace4b9..91509b9008 100644 --- a/.github/workflows/citation.yml +++ b/.github/workflows/citation.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out a copy of the repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Check whether the citation metadata from CITATION.cff is valid uses: citation-file-format/cffconvert-github-action@2.0.0 diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index 0965ea2f1c..8e99080608 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -28,10 +28,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d89791712f..5f03e854f6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -32,10 +32,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" diff --git a/.github/workflows/pre-commit-update.yml b/.github/workflows/pre-commit-update.yml index 1688f60d7e..b027befb20 100644 --- a/.github/workflows/pre-commit-update.yml +++ b/.github/workflows/pre-commit-update.yml @@ -11,9 +11,9 @@ jobs: auto-update: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 - uses: browniebroke/pre-commit-autoupdate-action@main @@ -21,7 +21,7 @@ jobs: run: | pre-commit run --all-files - - uses: peter-evans/create-pull-request@v5 + - uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.GITHUB_TOKEN }} branch: update/pre-commit-hooks diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index e9ba1a01bf..31d9577628 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -27,12 +27,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: recursive - name: Setup Python 3.10 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" @@ -40,6 +40,7 @@ jobs: run: | pip install pre-commit pre-commit install + pip install pylint - name: Run pre-commit run: | diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 37e15a4729..c94e246bbf 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -54,10 +54,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -108,10 +108,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Conda install - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true python-version: ${{ matrix.python-version }} @@ -148,10 +148,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/recent_reminder.yml b/.github/workflows/recent_reminder.yml new file mode 100644 index 0000000000..80fa5fa145 --- /dev/null +++ b/.github/workflows/recent_reminder.yml @@ -0,0 +1,46 @@ +name: Recent-Reminder + +on: + schedule: + - cron: '00 13 * * 1' + workflow_dispatch: + +jobs: + stale-reminder: + runs-on: ubuntu-latest + steps: + - name: Get cutoff dates + id: date + run: | + echo "RECENT_ISSUE_CUTOFF_DATE=$(date -d '-7 days' '+%Y-%m-%d')" >> $GITHUB_ENV + - name: Get list of issues that have had interactions in the last week + id: recent + uses: lee-dohm/select-matching-issues@v1 + with: + format: list + path: "recent_issues.md" + token: ${{ github.token }} + query: >- + is:issue + is:open + updated:>=${{ env.RECENT_ISSUE_CUTOFF_DATE }} + sort:updated-asc + - name: Combine issues into mail content + id: combine + run: | + echo "## Issues that have had interaction in the last 7 days
" >> mail.html + echo "$(" >> mail.html + - name: Send mail + id: mail + uses: dawidd6/action-send-mail@v3 + with: + server_address: ${{secrets.MAIL_SERVER_ADDRESS}} + server_port: ${{secrets.MAIL_SERVER_PORT}} + secure: true + username: ${{secrets.MAIL_USERNAME}} + password: ${{secrets.MAIL_PASSWORD}} + subject: '[Current SMAC3 Issues] Issues that have been interacted with since ${{ env.RECENT_ISSUE_CUTOFF_DATE }}' + to: ${{secrets.MAIL_TARGET}} + from: SMAC3 Stale-Bot <${{secrets.MAIL_ADDRESS}}> + html_body: file://mail.html + convert_markdown: true \ No newline at end of file diff --git a/.github/workflows/stale_reminder.yml b/.github/workflows/stale_reminder.yml deleted file mode 100644 index ce13276834..0000000000 --- a/.github/workflows/stale_reminder.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Stale-Reminder - -on: - schedule: - - cron: '00 13 * * 1' - workflow_dispatch: - -jobs: - stale-reminder: - runs-on: ubuntu-latest - steps: - - name: Get cutoff dates - id: date - run: | - echo "CUTOFF_DATE=$(date -d '-46 days' '+%Y-%m-%d')" >> $GITHUB_ENV - echo "RECENT_ISSUE_CUTOFF_DATE=$(date -d '-7 days' '+%Y-%m-%d')" >> $GITHUB_ENV - - name: Get list of issues that have had interactions in the last week - id: recent - uses: lee-dohm/select-matching-issues@v1 - with: - format: list - path: "recent_issues.md" - token: ${{ github.token }} - query: >- - is:issue - is:open - updated:>=${{ env.RECENT_ISSUE_CUTOFF_DATE }} - sort:updated-asc - - name: Collect issues that may become stale - id: stale - uses: lee-dohm/select-matching-issues@v1 - with: - format: list - path: "potentially_stale_issues.md" - token: ${{ github.token }} - query: >- - is:issue - is:open - -label:dependency,documentation,feature,enhancement,bug,test,example,discussion,duplicate,question - updated:<${{ env.CUTOFF_DATE }} - sort:updated-asc - - name: Collect labelled issues that have not had interaction in a long time (but will not become stale) - id: old - uses: lee-dohm/select-matching-issues@v1 - with: - format: list - path: "old_issues.md" - token: ${{ github.token }} - query: >- - is:issue - is:open - label:dependency,documentation,feature,enhancement,bug,test,example,discussion,duplicate,question - updated:<${{ env.CUTOFF_DATE }} - sort:updated-asc - - name: Combine issues into mail content - id: combine - run: | - echo "## Issues that have had interaction in the last 7 days
" >> mail.html - echo "$(" >> mail.html - echo "## Issues that may become stale in <= 14 days
" >> mail.html - echo "$(" >> mail.html - echo "## Issues that have not had interaction in the last 46 days but will not go stale due to their labels
" >> mail.html - echo "$(" >> mail.html - - name: Send mail - id: mail - uses: dawidd6/action-send-mail@v3 - with: - server_address: ${{secrets.MAIL_SERVER_ADDRESS}} - server_port: ${{secrets.MAIL_SERVER_PORT}} - secure: true - username: ${{secrets.MAIL_USERNAME}} - password: ${{secrets.MAIL_PASSWORD}} - subject: '[Stale Issues] Issues with last interaction before ${{ env.CUTOFF_DATE }}' - to: ${{secrets.MAIL_TARGET}} - from: SMAC3 Stale-Bot <${{secrets.MAIL_ADDRESS}}> - html_body: file://mail.html - convert_markdown: true \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dda7e5cae1..6b2d3934ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,3 +43,16 @@ repos: name: flake8 smac files: smac exclude: "scripts|tests" + + - repo: local + hooks: + - id: pylint + name: pylint + entry: pylint + language: system + types: [python] + args: + [ + "-rn", # Only display messages + "-sn", # Don't display the score + ] diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ff22b7210..6c268f0af7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,52 @@ +# 2.2.0 + +## Features +- Add example to specify total budget (fidelity units) instead of n_trials for multi-fidelity/Hyperband (#1121) + +## Dependencies +- Update numpy NaN (#1122) and restrict numpy version +- Upgrade to ConfigSpace 1.x.x (#1124) + +# 2.1.0 + +## Improvements +- Change the surrogate model to be retrained after every iteration by default in the case of blackbox optimization + (#1106). +- Integrate `LocalAndSortedPriorRandomSearch` functionality into `LocalAndSortedRandomSearch` (#1106). +- Change the way the `LocalAndSortedRandomSearch` works such that the incumbent always is a starting point and that + random configurations are sampled as the basis of the local search, not in addition (#1106). + +## Bugfixes +- Fix path for dask scheduler file (#1055). +- Add OrdinalHyperparameter for random forest imputer (#1065). +- Don't use mutable default argument (#1067). +- Propagate the Scenario random seed to `get_random_design` (#1066). +- Configurations that fail to become incumbents will be added to the rejected lists (#1069). +- SMAC RandomForest doesn't crash when `np.integer` used, i.e. as generated from a `np.random.RandomState` (#1084). +- Fix the handling of n_points/ challengers in the acquisition maximizers, such that this number now functions as the + number of points that are sampled from the acquisition function to find the next challengers. Now also doesn't + restrict the config selector to n_retrain many points for finding the max, and instead uses the defaults that are + defined via facades/ scenarios (#1106). + +## Misc +- ci: Update action version (#1072). + +## Minor +- When a custom dask client is provided, emit the warning that the `n_workers` parameter is ignored only if it deviates from its default value, `1` ([#1071](https://github.com/automl/SMAC3/pull/1071)). + # 2.0.2 +## Improvements +- Add an error when we get an empty dict data_to_scatter so that we can avoid an internal error caused in Dask precautiously. +- Add experimental instruction for installing SMAC in Windows via a WSL. +- More detailed documentation regarding continuing runs. +- Add a new example that demonstrates the use of intensification to speed up cross-validation for machine learning. + ## Bugfixes - Fix bug in the incumbent selection in the case that multi-fidelity is combined with multi-objective (#1019). +- Fix callback order (#1040). +- Handle configspace as dictionary in mlp and parego example. +- Adapt sgd loss to newest scikit-learn version. ## Features - Log to WandB (#1037) diff --git a/CITATION.cff b/CITATION.cff index 36435d6ecf..2279146d72 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,7 +9,7 @@ date-released: "2016-08-17" url: "https://automl.github.io/SMAC3/master/index.html" repository-code: "https://github.com/automl/SMAC3" -version: "2.0.1" +version: "2.2.0" type: "software" keywords: diff --git a/Makefile b/Makefile index f7727cf7f6..145d1bb14c 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ SHELL := /bin/bash NAME := SMAC3 PACKAGE_NAME := smac -VERSION := 2.0.1 +VERSION := 2.2.0 DIR := "${CURDIR}" SOURCE_DIR := ${PACKAGE_NAME} diff --git a/benchmark/src/benchmark.py b/benchmark/src/benchmark.py index 55cabd6d4e..9ae74a5fba 100644 --- a/benchmark/src/benchmark.py +++ b/benchmark/src/benchmark.py @@ -17,6 +17,8 @@ from collections import defaultdict from pathlib import Path +from smac.utils.numpyencoder import NumpyEncoder + import pandas as pd from src.tasks import TASKS # noqa: E402 from src.utils.exceptions import NotSupportedError # noqa: E402 @@ -79,7 +81,7 @@ def _save_data(self) -> None: """Saves the internal data to the file.""" print("Saving data...") with open(str(RAW_FILENAME), "w") as f: - json.dump(self._data, f, indent=4) + json.dump(self._data, f, indent=4, cls=NumpyEncoder) def _fill_keys(self) -> None: """Fill data with keys based on computer name, tasks, and selected version.""" diff --git a/benchmark/src/models/ac_branin.py b/benchmark/src/models/ac_branin.py index 12287520fb..e55c2862a8 100644 --- a/benchmark/src/models/ac_branin.py +++ b/benchmark/src/models/ac_branin.py @@ -20,7 +20,7 @@ def configspace(self) -> ConfigurationSpace: x2 = Float("x2", (0, 15), default=7.5) # Add hyperparameters and conditions to our configspace - cs.add_hyperparameters([x2]) + cs.add([x2]) return cs diff --git a/benchmark/src/models/branin.py b/benchmark/src/models/branin.py index 1fd20554fb..0f86d82ac6 100644 --- a/benchmark/src/models/branin.py +++ b/benchmark/src/models/branin.py @@ -20,7 +20,7 @@ def configspace(self) -> ConfigurationSpace: x2 = Float("x2", (0, 15), default=0) # Add hyperparameters and conditions to our configspace - cs.add_hyperparameters([x1, x2]) + cs.add([x1, x2]) return cs diff --git a/benchmark/src/models/himmelblau.py b/benchmark/src/models/himmelblau.py index cab99019a0..c12029e1ad 100644 --- a/benchmark/src/models/himmelblau.py +++ b/benchmark/src/models/himmelblau.py @@ -19,7 +19,7 @@ def configspace(self) -> ConfigurationSpace: y = Float("y", (-5, 5)) # Add hyperparameters and conditions to our configspace - cs.add_hyperparameters([x, y]) + cs.add([x, y]) return cs diff --git a/benchmark/src/models/mlp.py b/benchmark/src/models/mlp.py index 4867a2f42b..7329de0dfd 100644 --- a/benchmark/src/models/mlp.py +++ b/benchmark/src/models/mlp.py @@ -33,7 +33,7 @@ def configspace(self) -> ConfigurationSpace: learning_rate_init = Float("learning_rate_init", (0.0001, 1.0), default=0.001, log=True) # Add all hyperparameters at once: - cs.add_hyperparameters([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) + cs.add([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) # Adding conditions to restrict the hyperparameter space... # ... since learning rate is used when solver is 'sgd'. @@ -44,7 +44,7 @@ def configspace(self) -> ConfigurationSpace: use_batch_size = InCondition(child=batch_size, parent=solver, values=["sgd", "adam"]) # We can also add multiple conditions on hyperparameters at once: - cs.add_conditions([use_lr, use_batch_size, use_lr_init]) + cs.add([use_lr, use_batch_size, use_lr_init]) return cs diff --git a/benchmark/src/models/svm.py b/benchmark/src/models/svm.py index 88159ab709..3ed294ad0f 100644 --- a/benchmark/src/models/svm.py +++ b/benchmark/src/models/svm.py @@ -34,8 +34,8 @@ def configspace(self) -> ConfigurationSpace: use_gamma_value = InCondition(child=gamma_value, parent=gamma, values=["value"]) # Add hyperparameters and conditions to our configspace - cs.add_hyperparameters([kernel, C, shrinking, degree, coef, gamma, gamma_value]) - cs.add_conditions([use_degree, use_coef, use_gamma, use_gamma_value]) + cs.add([kernel, C, shrinking, degree, coef, gamma, gamma_value]) + cs.add([use_degree, use_coef, use_gamma, use_gamma_value]) return cs diff --git a/benchmark/src/wrappers/v20.py b/benchmark/src/wrappers/v20.py index 8ed0864ecb..3ee376a376 100644 --- a/benchmark/src/wrappers/v20.py +++ b/benchmark/src/wrappers/v20.py @@ -6,7 +6,7 @@ class Version20(Wrapper): - supported_versions: list[str] = ["2.0.1"] + supported_versions: list[str] = ["2.1.0"] def __init__(self, task: Task, seed: int) -> None: super().__init__(task, seed) diff --git a/docs/10_experimental.rst b/docs/10_experimental.rst new file mode 100644 index 0000000000..a075498c4a --- /dev/null +++ b/docs/10_experimental.rst @@ -0,0 +1,48 @@ +Experimental +============ + +.. warning:: + This part is experimental and might not work in each case. If you would like to suggest any changes, please let us know. + + +Installation in Windows via WSL +------------------------------ + +SMAC can be installed in a WSL (Windows Subsystem for Linux) under Windows. + +**1) Install WSL under Windows** + +Install WSL under Windows. This SMAC installation workflow was tested with Ubuntu 18.04. For Ubuntu 20.04, +it has been observed that the SMAC installation results in a segmentation fault (core dumped). + +**2) Get Anaconda** + +Download an Anaconda Linux version to drive D under Windows, e.g. D:\\Anaconda3-2023.03-1-Linux-x86_64 + +In the WSL, Windows resources are mounted under /mnt: + +.. code:: bash + + cd /mnt/d + bash Anaconda3-2023.03-1-Linux-x86_64 + +Enter this command to create the environment variable: + +.. code:: bash + + export PATH="$PATH:/home/${USER}/anaconda3/bin + +Input 'python' to check if the installation was successful. + +**3) Install SMAC** + +Change to your home folder and install the general software there: + +.. code:: bash + + cd /home/${USER} + sudo apt-get install software-properties-common + sudo apt-get update + sudo apt-get install build-essential swig + conda install gxx_linux-64 gcc_linux-64 swig + curl https://raw.githubusercontent.com/automl/smac3/master/requirements.txt | xargs -n 1 -L 1 pip install diff --git a/docs/1_installation.rst b/docs/1_installation.rst index 54f910a441..a835d9cf19 100644 --- a/docs/1_installation.rst +++ b/docs/1_installation.rst @@ -68,3 +68,11 @@ You must have `conda >= 4.9` installed. To update conda or check your current co Read `SMAC feedstock `_ for more details. + +Windows via WSL (Experimental) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +SMAC can be installed under Windows in a WSL (Windows Subsystem for Linux). +You can find an instruction on how to do this here: :ref:`Experimental`. +However, this is experimental and might not work in each case. +If you would like to suggest any changes, please let us know. diff --git a/docs/3_getting_started.rst b/docs/3_getting_started.rst index dc568184a7..dbc2873681 100644 --- a/docs/3_getting_started.rst +++ b/docs/3_getting_started.rst @@ -27,7 +27,7 @@ ranges and default values. "species": ["mouse", "cat", "dog"], # Categorical }) -Please see the documentation of `ConfigSpace `_ for more details. +Please see the documentation of `ConfigSpace `_ for more details. Target Function diff --git a/docs/advanced_usage/10_continue.rst b/docs/advanced_usage/10_continue.rst index 8a254c4e59..e29c3f9c96 100644 --- a/docs/advanced_usage/10_continue.rst +++ b/docs/advanced_usage/10_continue.rst @@ -1,14 +1,23 @@ Continue ======== -SMAC automatically restores states where it left off if a run was interrupted or finished. To do so, it reads in old -files (derived from scenario's name, output_directory and seed) and sets the components. +SMAC can automatically restore states where it left off if a run was interrupted or prematurely finished. To do so, +it reads in old files (derived from scenario's name, output_directory and seed) and obtains the scenario information +of the previous run from those to continue the run. + +The behavior can be controlled by setting the parameter ``overwrite`` in the facade to True or False, respectively: + +* If set to True, SMAC overwrites the run results if a previous run is found that is consistent in the meta data with the current setup. +* If set to False and a previous run is found that + + * is consistent in the meta data, the run is continued. + * is not consistent in the meta data, the user is asked for the exact behaviour (overwrite completely or rename old run first). .. warning:: - If you changed any code and specified a name, SMAC will ask you whether you still want to resume or - delete the old run completely. If you did not specify a name, SMAC generates a new name and the old run is - not affected. + If you changed any code affecting the run's meta data and specified a name, SMAC will ask you whether you still + want to overwrite the old run or rename the old run first. If you did not specify a name, SMAC generates a new name + and the old run is not affected. Please have a look at our :ref:`continue example`. \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index d7b5598884..4adbf4cd28 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,9 @@ "version": version, "versions": { f"v{version}": "#", + "v2.2.0": "https://automl.github.io/SMAC3/v2.2.0/", + "v2.1.0": "https://automl.github.io/SMAC3/v2.1.0/", + "v2.0.1": "https://automl.github.io/SMAC3/v2.0.1/", "v2.0.0": "https://automl.github.io/SMAC3/v2.0.0/", "v2.0.0b1": "https://automl.github.io/SMAC3/v2.0.0b1/", "v2.0.0a2": "https://automl.github.io/SMAC3/v2.0.0a2/", diff --git a/examples/1_basics/1_quadratic_function.py b/examples/1_basics/1_quadratic_function.py index 3cafb846b8..4d27c7ae6d 100644 --- a/examples/1_basics/1_quadratic_function.py +++ b/examples/1_basics/1_quadratic_function.py @@ -26,7 +26,7 @@ class QuadraticFunction: def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x = Float("x", (-5, 5), default=-5) - cs.add_hyperparameters([x]) + cs.add([x]) return cs diff --git a/examples/1_basics/2_svm_cv.py b/examples/1_basics/2_svm_cv.py index cfe39d4bb1..345fcffb07 100644 --- a/examples/1_basics/2_svm_cv.py +++ b/examples/1_basics/2_svm_cv.py @@ -46,8 +46,8 @@ def configspace(self) -> ConfigurationSpace: use_gamma_value = InCondition(child=gamma_value, parent=gamma, values=["value"]) # Add hyperparameters and conditions to our configspace - cs.add_hyperparameters([kernel, C, shrinking, degree, coef, gamma, gamma_value]) - cs.add_conditions([use_degree, use_coef, use_gamma, use_gamma_value]) + cs.add([kernel, C, shrinking, degree, coef, gamma, gamma_value]) + cs.add([use_degree, use_coef, use_gamma, use_gamma_value]) return cs diff --git a/examples/1_basics/3_ask_and_tell.py b/examples/1_basics/3_ask_and_tell.py index 5d0e5e78c5..6ab8b5ba80 100644 --- a/examples/1_basics/3_ask_and_tell.py +++ b/examples/1_basics/3_ask_and_tell.py @@ -20,7 +20,7 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x0 = Float("x0", (-5, 10), default=-3) x1 = Float("x1", (-5, 10), default=-4) - cs.add_hyperparameters([x0, x1]) + cs.add([x0, x1]) return cs diff --git a/examples/1_basics/4_callback.py b/examples/1_basics/4_callback.py index c3d66a4b94..0fd9e9d9d7 100644 --- a/examples/1_basics/4_callback.py +++ b/examples/1_basics/4_callback.py @@ -27,7 +27,7 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x0 = Float("x0", (-5, 10), default=-3) x1 = Float("x1", (-5, 10), default=-4) - cs.add_hyperparameters([x0, x1]) + cs.add([x0, x1]) return cs diff --git a/examples/1_basics/5_continue.py b/examples/1_basics/5_continue.py index 04cca4aed9..63cfb3957f 100644 --- a/examples/1_basics/5_continue.py +++ b/examples/1_basics/5_continue.py @@ -2,8 +2,16 @@ Continue an Optimization ^^^^^^^^^^^^^^^^^^^^^^^^ -SMAC can also be continued. In this example, an optimization of a simple quadratic -function is continued. We use a custom callback, to artificially stop the first optimization. +SMAC can also be continued from a previous run. To do so, it reads in old files (derived from scenario's name, +output_directory and seed) and sets the corresponding components. In this example, an optimization of a simple quadratic +function is continued. + +First, after creating a scenario with 50 trials, we run SMAC with overwrite=True. This will +overwrite any previous runs (in case the example was called before). We use a custom callback to artificially stop +this first optimization after 10 trials. + +Second, we again run the SMAC optimization using the same scenario, but this time with overwrite=False. As +there already is a previous run with the same meta data, this run will be continued until the 50 trials are reached. """ from __future__ import annotations @@ -39,7 +47,7 @@ class QuadraticFunction: def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x = Float("x", (-5, 5), default=-5) - cs.add_hyperparameters([x]) + cs.add([x]) return cs diff --git a/examples/1_basics/6_priors.py b/examples/1_basics/6_priors.py index 218bd8f460..691460c0b2 100644 --- a/examples/1_basics/6_priors.py +++ b/examples/1_basics/6_priors.py @@ -95,13 +95,13 @@ def configspace(self) -> ConfigurationSpace: "learning_rate_init", lower=1e-5, upper=1.0, - mu=np.log(1e-3), - sigma=np.log(10), + mu=1e-3, # will be transformed to log space later + sigma=10, # will be transformed to log space later log=True, ) # Add all hyperparameters at once: - cs.add_hyperparameters([n_layer, n_neurons, activation, optimizer, batch_size, learning_rate_init]) + cs.add([n_layer, n_neurons, activation, optimizer, batch_size, learning_rate_init]) return cs diff --git a/examples/1_basics/7_parallelization_cluster.py b/examples/1_basics/7_parallelization_cluster.py index 36f79586e3..2467f7d229 100644 --- a/examples/1_basics/7_parallelization_cluster.py +++ b/examples/1_basics/7_parallelization_cluster.py @@ -6,6 +6,9 @@ SLURM cluster. If you do not want to use a cluster but your local machine, set dask_client to `None` and pass `n_workers` to the `Scenario`. +Sometimes, the submitted jobs by the slurm client might be cancelled once it starts. In that +case, you could try to start your job from a computing node + :warning: On some clusters you cannot spawn new jobs when running a SLURMCluster inside a job instead of on the login node. No obvious errors might be raised but it can hang silently. @@ -41,7 +44,7 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x0 = Float("x0", (-5, 10), default=-5, log=False) x1 = Float("x1", (0, 15), default=2, log=False) - cs.add_hyperparameters([x0, x1]) + cs.add([x0, x1]) return cs @@ -77,7 +80,7 @@ def train(self, config: Configuration, seed: int = 0) -> float: model = Branin() # Scenario object specifying the optimization "environment" - scenario = Scenario(model.configspace, deterministic=True, n_trials=100) + scenario = Scenario(model.configspace, deterministic=True, n_trials=100, trial_walltime_limit=100) # Create cluster n_workers = 4 # Use 4 workers on the cluster @@ -97,6 +100,10 @@ def train(self, config: Configuration, seed: int = 0) -> float: walltime="00:10:00", processes=1, log_directory="tmp/smac_dask_slurm", + # if you would like to limit the resources consumption of each function evaluation with pynisher, you need to + # set nanny as False + # Otherwise, an error `daemonic processes are not allowed to have children` will raise! + nanny=False, # if you do not use pynisher to limit the memory/time usage, feel free to set this one as True ) cluster.scale(jobs=n_workers) diff --git a/examples/2_multi_fidelity/1_mlp_epochs.py b/examples/2_multi_fidelity/1_mlp_epochs.py index 9fd256c5d6..5cb0aefa05 100644 --- a/examples/2_multi_fidelity/1_mlp_epochs.py +++ b/examples/2_multi_fidelity/1_mlp_epochs.py @@ -65,7 +65,7 @@ def configspace(self) -> ConfigurationSpace: learning_rate_init = Float("learning_rate_init", (0.0001, 1.0), default=0.001, log=True) # Add all hyperparameters at once: - cs.add_hyperparameters([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) + cs.add([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) # Adding conditions to restrict the hyperparameter space... # ... since learning rate is only used when solver is 'sgd'. @@ -76,7 +76,7 @@ def configspace(self) -> ConfigurationSpace: use_batch_size = InCondition(child=batch_size, parent=solver, values=["sgd", "adam"]) # We can also add multiple conditions on hyperparameters at once: - cs.add_conditions([use_lr, use_batch_size, use_lr_init]) + cs.add([use_lr, use_batch_size, use_lr_init]) return cs @@ -84,9 +84,9 @@ def train(self, config: Configuration, seed: int = 0, budget: int = 25) -> float # For deactivated parameters (by virtue of the conditions), # the configuration stores None-values. # This is not accepted by the MLP, so we replace them with placeholder values. - lr = config["learning_rate"] if config["learning_rate"] else "constant" - lr_init = config["learning_rate_init"] if config["learning_rate_init"] else 0.001 - batch_size = config["batch_size"] if config["batch_size"] else 200 + lr = config.get("learning_rate", "constant") + lr_init = config.get("learning_rate_init", 0.001) + batch_size = config.get("batch_size", 200) with warnings.catch_warnings(): warnings.filterwarnings("ignore") diff --git a/examples/2_multi_fidelity/2_sgd_datasets.py b/examples/2_multi_fidelity/2_sgd_datasets.py index 384d1c2246..178ea21c2b 100644 --- a/examples/2_multi_fidelity/2_sgd_datasets.py +++ b/examples/2_multi_fidelity/2_sgd_datasets.py @@ -76,7 +76,7 @@ def configspace(self) -> ConfigurationSpace: learning_rate = Categorical("learning_rate", ["constant", "invscaling", "adaptive"], default="constant") eta0 = Float("eta0", (0.00001, 1), default=0.1, log=True) # Add the parameters to configuration space - cs.add_hyperparameters([alpha, l1_ratio, learning_rate, eta0]) + cs.add([alpha, l1_ratio, learning_rate, eta0]) return cs @@ -89,7 +89,7 @@ def train(self, config: Configuration, instance: str, seed: int = 0) -> float: # SGD classifier using given configuration clf = SGDClassifier( - loss="log", + loss="log_loss", penalty="elasticnet", alpha=config["alpha"], l1_ratio=config["l1_ratio"], diff --git a/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py b/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py new file mode 100644 index 0000000000..7c0ebdcf0d --- /dev/null +++ b/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py @@ -0,0 +1,112 @@ +""" +Specify Number of Trials via a Total Budget in Hyperband +^^^^^^^^^^^^^^^^^^ +This example uses a dummy function but illustrates how to setup Hyperband if you +want to specify a total optimization budget in terms of fidelity units. + +In Hyperband, normally SMAC calculates a typical Hyperband round. +If the number of trials is not used up by one single round, the next round is started. +Instead of specifying the number of trial beforehand, specify the total budget +in terms of the fidelity units and let SMAC calculate how many trials that would be. + + +""" +from __future__ import annotations + +import numpy as np +from ConfigSpace import Configuration, ConfigurationSpace, Float +from matplotlib import pyplot as plt + +from smac import MultiFidelityFacade, RunHistory, Scenario +from smac.intensifier.hyperband_utils import get_n_trials_for_hyperband_multifidelity + +__copyright__ = "Copyright 2021, AutoML.org Freiburg-Hannover" +__license__ = "3-clause BSD" + + +class QuadraticFunction: + max_budget = 500 + + @property + def configspace(self) -> ConfigurationSpace: + cs = ConfigurationSpace(seed=0) + x = Float("x", (-5, 5), default=-5) + cs.add([x]) + + return cs + + def train(self, config: Configuration, seed: int = 0, budget: float | None = None) -> float: + """Returns the y value of a quadratic function with a minimum we know to be at x=0.""" + x = config["x"] + + if budget is None: + multiplier = 1 + else: + multiplier = 1 + budget / self.max_budget + + return x**2 * multiplier + + +def plot(runhistory: RunHistory, incumbent: Configuration) -> None: + plt.figure() + + # Plot ground truth + x = list(np.linspace(-5, 5, 100)) + y = [xi * xi for xi in x] + plt.plot(x, y) + + # Plot all trials + for k, v in runhistory.items(): + config = runhistory.get_config(k.config_id) + x = config["x"] + y = v.cost # type: ignore + plt.scatter(x, y, c="blue", alpha=0.1, zorder=9999, marker="o") + + # Plot incumbent + plt.scatter(incumbent["x"], incumbent["x"] * incumbent["x"], c="red", zorder=10000, marker="x") + + plt.show() + + +if __name__ == "__main__": + model = QuadraticFunction() + + min_budget = 10 # minimum budget per trial + max_budget = 500 # maximum budget per trial + eta = 3 # standard HB parameter influencing the number of stages + + # Let's calculate how many trials we need to exhaust the total optimization budget (in terms of + # fidelity units) + n_trials = get_n_trials_for_hyperband_multifidelity( + total_budget=10000, # this is the total optimization budget we specify in terms of fidelity units + min_budget=min_budget, # This influences the Hyperband rounds, minimum budget per trial + max_budget=max_budget, # This influences the Hyperband rounds, maximum budget per trial + eta=eta, # This influences the Hyperband rounds + print_summary=True, + ) + + # Scenario object specifying the optimization "environment" + scenario = Scenario( + model.configspace, deterministic=True, n_trials=n_trials, min_budget=min_budget, max_budget=max_budget + ) + + # Now we use SMAC to find the best hyperparameters + smac = MultiFidelityFacade( + scenario, + model.train, # We pass the target function here + overwrite=True, # Overrides any previous results that are found that are inconsistent with the meta-data + intensifier=MultiFidelityFacade.get_intensifier(scenario=scenario, eta=eta), + ) + + incumbent = smac.optimize() + + # Get cost of default configuration + default_cost = smac.validate(model.configspace.get_default_configuration()) + print(f"Default cost: {default_cost}") + + # Let's calculate the cost of the incumbent + incumbent_cost = smac.validate(incumbent) + print(f"Incumbent cost: {incumbent_cost}") + + # Let's plot it too + plot(smac.runhistory, incumbent) diff --git a/examples/3_multi_objective/2_parego.py b/examples/3_multi_objective/2_parego.py index d8fdc5ff8d..b5294fb98b 100644 --- a/examples/3_multi_objective/2_parego.py +++ b/examples/3_multi_objective/2_parego.py @@ -54,21 +54,21 @@ def configspace(self) -> ConfigurationSpace: learning_rate = Categorical("learning_rate", ["constant", "invscaling", "adaptive"], default="constant") learning_rate_init = Float("learning_rate_init", (0.0001, 1.0), default=0.001, log=True) - cs.add_hyperparameters([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) + cs.add([n_layer, n_neurons, activation, solver, batch_size, learning_rate, learning_rate_init]) use_lr = EqualsCondition(child=learning_rate, parent=solver, value="sgd") use_lr_init = InCondition(child=learning_rate_init, parent=solver, values=["sgd", "adam"]) use_batch_size = InCondition(child=batch_size, parent=solver, values=["sgd", "adam"]) # We can also add multiple conditions on hyperparameters at once: - cs.add_conditions([use_lr, use_batch_size, use_lr_init]) + cs.add([use_lr, use_batch_size, use_lr_init]) return cs def train(self, config: Configuration, seed: int = 0, budget: int = 10) -> dict[str, float]: - lr = config["learning_rate"] if config["learning_rate"] else "constant" - lr_init = config["learning_rate_init"] if config["learning_rate_init"] else 0.001 - batch_size = config["batch_size"] if config["batch_size"] else 200 + lr = config.get("learning_rate", "constant") + lr_init = config.get("learning_rate_init", 0.001) + batch_size = config.get("batch_size", 200) start_time = time.time() diff --git a/examples/4_advanced_optimizer/1_turbo_optimizer.py b/examples/4_advanced_optimizer/1_turbo_optimizer.py index 860243c028..dc936f7079 100644 --- a/examples/4_advanced_optimizer/1_turbo_optimizer.py +++ b/examples/4_advanced_optimizer/1_turbo_optimizer.py @@ -28,7 +28,7 @@ # cs = ConfigurationSpace(seed=0) # x0 = Float("x0", (-5, 10), default=-3) # x1 = Float("x1", (-5, 10), default=-4) -# cs.add_hyperparameters([x0, x1]) +# cs.add([x0, x1]) # return cs diff --git a/examples/4_advanced_optimizer/2_boing_optimizer.py b/examples/4_advanced_optimizer/2_boing_optimizer.py index 3eb1a18691..815b1f3401 100644 --- a/examples/4_advanced_optimizer/2_boing_optimizer.py +++ b/examples/4_advanced_optimizer/2_boing_optimizer.py @@ -27,7 +27,7 @@ # cs = ConfigurationSpace(seed=0) # x0 = Float("x0", (-5, 10), default=-3) # x1 = Float("x1", (-5, 10), default=-4) -# cs.add_hyperparameters([x0, x1]) +# cs.add([x0, x1]) # return cs diff --git a/examples/4_advanced_optimizer/3_metadata_callback.py b/examples/4_advanced_optimizer/3_metadata_callback.py index ab35f28627..b82670dbc6 100644 --- a/examples/4_advanced_optimizer/3_metadata_callback.py +++ b/examples/4_advanced_optimizer/3_metadata_callback.py @@ -36,7 +36,7 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x0 = Float("x0", (-5, 10), default=-3) x1 = Float("x1", (-5, 10), default=-4) - cs.add_hyperparameters([x0, x1]) + cs.add([x0, x1]) return cs diff --git a/examples/4_advanced_optimizer/4_intensify_crossvalidation.py b/examples/4_advanced_optimizer/4_intensify_crossvalidation.py new file mode 100644 index 0000000000..679253da18 --- /dev/null +++ b/examples/4_advanced_optimizer/4_intensify_crossvalidation.py @@ -0,0 +1,126 @@ +""" +Speeding up Cross-Validation with Intensification +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +An example of optimizing a simple support vector machine on the digits dataset. In contrast to the +[simple example](examples/1_basics/2_svm_cv.py), in which all cross-validation folds are executed +at once, we use the intensification mechanism described in the original +[SMAC paper](https://link.springer.com/chapter/10.1007/978-3-642-25566-3_40) as also demonstrated +by [Auto-WEKA](https://dl.acm.org/doi/10.1145/2487575.2487629). This mechanism allows us to +terminate the evaluation of a configuration if after a certain number of folds, the configuration +is found to be worse than the incumbent configuration. This is especially useful if the evaluation +of a configuration is expensive, e.g., if we have to train a neural network or if we have to +evaluate the configuration on a large dataset. +""" +__copyright__ = "Copyright 2023, AutoML.org Freiburg-Hannover" +__license__ = "3-clause BSD" + +N_FOLDS = 10 # Global variable that determines the number of folds + +from ConfigSpace import Configuration, ConfigurationSpace, Float +from sklearn import datasets, svm +from sklearn.model_selection import StratifiedKFold + +from smac import HyperparameterOptimizationFacade, Scenario +from smac.intensifier import Intensifier + +# We load the digits dataset, a small-scale 10-class digit recognition dataset +X, y = datasets.load_digits(return_X_y=True) + + +class SVM: + @property + def configspace(self) -> ConfigurationSpace: + # Build Configuration Space which defines all parameters and their ranges + cs = ConfigurationSpace(seed=0) + + # First we create our hyperparameters + C = Float("C", (2**-5, 2**15), default=1.0, log=True) + gamma = Float("gamma", (2**-15, 2**3), default=1.0, log=True) + + # Add hyperparameters to our configspace + cs.add([C, gamma]) + + return cs + + def train(self, config: Configuration, instance: str, seed: int = 0) -> float: + """Creates a SVM based on a configuration and evaluate on the given fold of the digits dataset + + Parameters + ---------- + config: Configuration + The configuration to train the SVM. + instance: str + The name of the instance this configuration should be evaluated on. This is always of type + string by definition. In our case we cast to int, but this could also be the filename of a + problem instance to be loaded. + seed: int + The seed used for this call. + """ + instance = int(instance) + classifier = svm.SVC(**config, random_state=seed) + splitter = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=seed) + for k, (train_idx, test_idx) in enumerate(splitter.split(X=X, y=y)): + if k != instance: + continue + else: + train_X = X[train_idx] + train_y = y[train_idx] + test_X = X[test_idx] + test_y = y[test_idx] + classifier.fit(train_X, train_y) + cost = 1 - classifier.score(test_X, test_y) + + return cost + + +if __name__ == "__main__": + classifier = SVM() + + # Next, we create an object, holding general information about the run + scenario = Scenario( + classifier.configspace, + n_trials=50, # We want to run max 50 trials (combination of config and instances in the case of + # deterministic=True. In the case of deterministic=False, this would be the + # combination of instances, seeds and configs). The number of distinct configurations + # evaluated by SMAC will be lower than this number because some of the configurations + # will be executed on more than one instance (CV fold). + instances=[f"{i}" for i in range(N_FOLDS)], # Specify all instances by their name (as a string) + instance_features={f"{i}": [i] for i in range(N_FOLDS)}, # breaks SMAC + deterministic=True # To simplify the problem we make SMAC believe that we have a deterministic + # optimization problem. + ) + + # We want to run the facade's default initial design, but we want to change the number + # of initial configs to 5. + initial_design = HyperparameterOptimizationFacade.get_initial_design(scenario, n_configs=5) + + # Now we use SMAC to find the best hyperparameters + smac = HyperparameterOptimizationFacade( + scenario, + classifier.train, + initial_design=initial_design, + overwrite=True, # If the run exists, we overwrite it; alternatively, we can continue from last state + # The next line defines the intensifier, i.e., the module that governs the selection of + # instance-seed pairs. Since we set deterministic to True above, it only governs the instance in + # this example. Technically, it is not necessary to create the intensifier as a user, but it is + # necessary to do so because we change the argument max_config_calls (the number of instance-seed pairs + # per configuration to try) to the number of cross-validation folds, while the default would be 3. + intensifier=Intensifier(scenario=scenario, max_config_calls=N_FOLDS, seed=0), + ) + + incumbent = smac.optimize() + + # Get cost of default configuration + default_cost = smac.validate(classifier.configspace.get_default_configuration()) + print(f"Default cost: {default_cost}") + + # Let's calculate the cost of the incumbent + incumbent_cost = smac.validate(incumbent) + print(f"Incumbent cost: {incumbent_cost}") + + # Let's see how many configurations we have evaluated. If this number is higher than 5, we have looked + # at more configurations than would have been possible with regular cross-validation, where the number + # of configurations would be determined by the number of trials divided by the number of folds (50 / 10). + runhistory = smac.runhistory + print(f"Number of evaluated configurations: {len(runhistory.config_ids)}") diff --git a/pyproject.toml b/pyproject.toml index c0e1ae4ed9..77783d8a77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,11 @@ add-ignore = [ # http://www.pydocstyle.org/en/stable/error_codes.html "D415", # First line should end with a period, question mark, or exclamation point ] +[tool.pylint."messages control"] +# FIXME: This is to do a staged introduction of pylint checks for just a single class of problems initially (#1067). +disable = ["all"] +enable = ["dangerous-default-value"] + [tool.mypy] python_version = "3.9" show_error_codes = true diff --git a/setup.py b/setup.py index 804413688a..e15b2f44c3 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ def read_file(filepath: str) -> str: "pydocstyle", "flake8", "pre-commit", + "pylint", ], "wandb": [ "wandb", @@ -59,16 +60,16 @@ def read_file(filepath: str) -> str: include_package_data=True, python_requires=">=3.8", install_requires=[ - "numpy>=1.23.3", + "numpy>=1.23.3,<2.0.0", "scipy>=1.9.2", "psutil", "pynisher>=1.0.0", - "ConfigSpace>=0.6.1", + "ConfigSpace>=1.0.0", "joblib", "scikit-learn>=1.1.2", "pyrfr>=0.9.0", "dask[distributed]", - "dask_jobqueue", + "dask_jobqueue>=0.8.2", "emcee>=3.0.0", "regex", "pyyaml", diff --git a/smac/__init__.py b/smac/__init__.py index f3fe5805b9..439dc48fae 100644 --- a/smac/__init__.py +++ b/smac/__init__.py @@ -12,14 +12,14 @@ description = "SMAC3, a Python implementation of 'Sequential Model-based Algorithm Configuration'." url = "https://www.automl.org/" project_urls = { - "Documentation": "https://https://github.com/automl.github.io/SMAC3/main", - "Source Code": "https://github.com/https://github.com/automl/smac", + "Documentation": "https://automl.github.io/SMAC3/main", + "Source Code": "https://github.com/automl/SMAC3", } copyright = f""" Copyright {datetime.date.today().strftime('%Y')}, Marius Lindauer, Katharina Eggensperger, Matthias Feurer, André Biedenkapp, Difan Deng, Carolin Benjamins, Tim Ruhkopf, René Sass and Frank Hutter""" -version = "2.0.1" +version = "2.2.0" try: diff --git a/smac/acquisition/maximizer/__init__.py b/smac/acquisition/maximizer/__init__.py index 6fea59b459..5e3756190e 100644 --- a/smac/acquisition/maximizer/__init__.py +++ b/smac/acquisition/maximizer/__init__.py @@ -3,7 +3,6 @@ ) from smac.acquisition.maximizer.differential_evolution import DifferentialEvolution from smac.acquisition.maximizer.local_and_random_search import ( - LocalAndSortedPriorRandomSearch, LocalAndSortedRandomSearch, ) from smac.acquisition.maximizer.local_search import LocalSearch @@ -13,7 +12,6 @@ "AbstractAcquisitionMaximizer", "DifferentialEvolution", "LocalAndSortedRandomSearch", - "LocalAndSortedPriorRandomSearch", "LocalSearch", "RandomSearch", ] diff --git a/smac/acquisition/maximizer/abstract_acqusition_maximizer.py b/smac/acquisition/maximizer/abstract_acqusition_maximizer.py index 5a14a88141..e148cb6ede 100644 --- a/smac/acquisition/maximizer/abstract_acqusition_maximizer.py +++ b/smac/acquisition/maximizer/abstract_acqusition_maximizer.py @@ -27,12 +27,9 @@ class AbstractAcquisitionMaximizer: Parameters ---------- - configspace : ConfigurationSpace - acquisition_function : AbstractAcquisitionFunction - challengers : int, defaults to 5000 - Number of configurations to sample from the configuration space to get - the acquisition function value for, thus challenging the current - incumbent and becoming a candidate for the next function evaluation. + configspace : ConfigurationSpace acquisition_function : AbstractAcquisitionFunction + challengers : int, defaults to 5000 Number of configurations sampled during the optimization process, + details depend on the used maximizer. Also, the number of configurations that is returned by calling `maximize`. seed : int, defaults to 0 """ @@ -85,8 +82,8 @@ def maximize( previous_configs: list[Configuration] Previous evaluated configurations. n_points: int, defaults to None - Number of points to be sampled. If `n_points` is not specified, - `self._challengers` is used. + Number of points to be sampled & number of configurations to be returned. If `n_points` is not specified, + `self._challengers` is used. Semantics depend on concrete implementation. random_design: AbstractRandomDesign, defaults to None Part of the returned ChallengerList such that we can interleave random configurations by a scheme defined by the random design. The method `random_design.next_iteration()` diff --git a/smac/acquisition/maximizer/differential_evolution.py b/smac/acquisition/maximizer/differential_evolution.py index d1201c19c6..0f2ce15e60 100644 --- a/smac/acquisition/maximizer/differential_evolution.py +++ b/smac/acquisition/maximizer/differential_evolution.py @@ -30,6 +30,7 @@ def _maximize( previous_configs: list[Configuration], n_points: int, ) -> list[tuple[float, Configuration]]: + # n_points is not used here, but is required by the interface configs: list[tuple[float, Configuration]] = [] diff --git a/smac/acquisition/maximizer/helpers.py b/smac/acquisition/maximizer/helpers.py index d7319f20fb..db06e2aa46 100644 --- a/smac/acquisition/maximizer/helpers.py +++ b/smac/acquisition/maximizer/helpers.py @@ -4,8 +4,8 @@ from ConfigSpace import Configuration, ConfigurationSpace +from smac.random_design import ProbabilityRandomDesign from smac.random_design.abstract_random_design import AbstractRandomDesign -from smac.random_design.modulus_design import ModulusRandomDesign class ChallengerList(Iterator): @@ -20,7 +20,7 @@ class ChallengerList(Iterator): ---------- configspace : ConfigurationSpace challenger_callback : Callable - Callback function which returns a list of challengers (without interleaved random configurations, must a be a + Callback function which returns a list of challengers (without interleaved random configurations), must a be a python closure. random_design : AbstractRandomDesign | None, defaults to ModulusRandomDesign(modulus=2.0) Which random design should be used. @@ -30,7 +30,7 @@ def __init__( self, configspace: ConfigurationSpace, challenger_callback: Callable, - random_design: AbstractRandomDesign | None = ModulusRandomDesign(modulus=2.0), + random_design: AbstractRandomDesign | None = ProbabilityRandomDesign(seed=0, probability=0.08447232371720552), ): self._challengers_callback = challenger_callback self._challengers: list[Configuration] | None = None @@ -72,44 +72,3 @@ def __len__(self) -> int: self._challengers = self._challengers_callback() return len(self._challengers) - self._index - - -''' -class FixedSet(AbstractAcquisitionMaximizer): - def __init__( - self, - configurations: list[Configuration], - acquisition_function: AbstractAcquisitionFunction, - configspace: ConfigurationSpace, - challengers: int = 5000, - seed: int = 0, - ): - """Maximize the acquisition function over a finite list of configurations. - - Parameters - ---------- - configurations : list[~smac._configspace.Configuration] - Candidate configurations - acquisition_function : ~smac.acquisition.AbstractAcquisitionFunction - - configspace : ~smac._configspace.ConfigurationSpace - - rng : np.random.RandomState or int, optional - """ - super().__init__( - acquisition_function=acquisition_function, configspace=configspace, challengers=challengers, seed=seed - ) - self.configurations = configurations - - def _maximize( - self, - runhistory: RunHistory, - stats: Stats, - n_points: int, - ) -> list[tuple[float, Configuration]]: - configurations = copy.deepcopy(self.configurations) - for config in configurations: - config.origin = "Fixed Set" - - return self._sort_by_acquisition_value(configurations) -''' diff --git a/smac/acquisition/maximizer/local_and_random_search.py b/smac/acquisition/maximizer/local_and_random_search.py index bb6daaef51..71c7f86c47 100644 --- a/smac/acquisition/maximizer/local_and_random_search.py +++ b/smac/acquisition/maximizer/local_and_random_search.py @@ -21,7 +21,7 @@ class LocalAndSortedRandomSearch(AbstractAcquisitionMaximizer): """Implement SMAC's default acquisition function optimization. - This optimizer performs local search from the previous best points according, to the acquisition + This optimizer performs local search from the previous best points according to the acquisition function, uses the acquisition function to sort randomly sampled configurations. Random configurations are interleaved by the main SMAC code. @@ -31,6 +31,10 @@ class LocalAndSortedRandomSearch(AbstractAcquisitionMaximizer): Parameters ---------- configspace : ConfigurationSpace + uniform_configspace : ConfigurationSpace + A version of the user-defined ConfigurationSpace where all parameters are uniform (or have their weights removed + in the case of a categorical hyperparameter). Can optionally be given and sampling ratios be defined via the + `prior_sampling_fraction` parameter. acquisition_function : AbstractAcquisitionFunction | None, defaults to None challengers : int, defaults to 5000 Number of challengers. @@ -40,6 +44,9 @@ class LocalAndSortedRandomSearch(AbstractAcquisitionMaximizer): [LocalSearch] number of steps during a plateau walk before local search terminates. local_search_iterations: int, defauts to 10 [Local Search] number of local search iterations. + prior_sampling_fraction: float, defaults to 0.5 + The ratio of random samples that are taken from the user-defined ConfigurationSpace, as opposed to the uniform + version (needs `uniform_configspace`to be defined). seed : int, defaults to 0 """ @@ -52,6 +59,8 @@ def __init__( n_steps_plateau_walk: int = 10, local_search_iterations: int = 10, seed: int = 0, + uniform_configspace: ConfigurationSpace | None = None, + prior_sampling_fraction: float | None = None, ) -> None: super().__init__( configspace, @@ -60,11 +69,28 @@ def __init__( seed=seed, ) - self._random_search = RandomSearch( - configspace=configspace, - acquisition_function=acquisition_function, - seed=seed, - ) + if uniform_configspace is not None and prior_sampling_fraction is None: + prior_sampling_fraction = 0.5 + if uniform_configspace is None and prior_sampling_fraction is not None: + raise ValueError("If `prior_sampling_fraction` is given, `uniform_configspace` must be defined.") + if uniform_configspace is not None and prior_sampling_fraction is not None: + self._prior_random_search = RandomSearch( + acquisition_function=acquisition_function, + configspace=configspace, + seed=seed, + ) + + self._uniform_random_search = RandomSearch( + acquisition_function=acquisition_function, + configspace=uniform_configspace, + seed=seed, + ) + else: + self._random_search = RandomSearch( + configspace=configspace, + acquisition_function=acquisition_function, + seed=seed, + ) self._local_search = LocalSearch( configspace=configspace, @@ -75,6 +101,8 @@ def __init__( ) self._local_search_iterations = local_search_iterations + self._prior_sampling_fraction = prior_sampling_fraction + self._uniform_configspace = uniform_configspace @property def acquisition_function(self) -> AbstractAcquisitionFunction | None: # noqa: D102 @@ -84,18 +112,31 @@ def acquisition_function(self) -> AbstractAcquisitionFunction | None: # noqa: D @acquisition_function.setter def acquisition_function(self, acquisition_function: AbstractAcquisitionFunction) -> None: self._acquisition_function = acquisition_function - self._random_search._acquisition_function = acquisition_function + if self._uniform_configspace is not None: + self._prior_random_search._acquisition_function = acquisition_function + self._uniform_random_search._acquisition_function = acquisition_function + else: + self._random_search._acquisition_function = acquisition_function self._local_search._acquisition_function = acquisition_function @property def meta(self) -> dict[str, Any]: # noqa: D102 meta = super().meta - meta.update( - { - "random_search": self._random_search.meta, - "local_search": self._local_search.meta, - } - ) + if self._uniform_configspace is None: + meta.update( + { + "random_search": self._random_search.meta, + "local_search": self._local_search.meta, + } + ) + else: + meta.update( + { + "prior_random_search": self._prior_random_search.meta, + "uniform_random_search": self._uniform_random_search.meta, + "local_search": self._local_search.meta, + } + ) return meta @@ -104,148 +145,45 @@ def _maximize( previous_configs: list[Configuration], n_points: int, ) -> list[tuple[float, Configuration]]: - - # Get configurations sorted by EI - next_configs_by_random_search_sorted = self._random_search._maximize( - previous_configs=previous_configs, - n_points=n_points, - _sorted=True, - ) - + if self._uniform_configspace is not None and self._prior_sampling_fraction is not None: + # Get configurations sorted by acquisition function value + next_configs_by_prior_random_search_sorted = self._prior_random_search._maximize( + previous_configs, + round(n_points * self._prior_sampling_fraction), + _sorted=True, + ) + + # Get configurations sorted by acquisition function value + next_configs_by_uniform_random_search_sorted = self._uniform_random_search._maximize( + previous_configs, + round(n_points * (1 - self._prior_sampling_fraction)), + _sorted=True, + ) + next_configs_by_random_search_sorted = ( + next_configs_by_uniform_random_search_sorted + next_configs_by_prior_random_search_sorted + ) + next_configs_by_random_search_sorted.sort(reverse=True, key=lambda x: x[0]) + else: + # Get configurations sorted by acquisition function value + next_configs_by_random_search_sorted = self._random_search._maximize( + previous_configs=previous_configs, + n_points=n_points, + _sorted=True, + ) + + # Choose the best self._local_search_iterations random configs to start the local search, and choose only + # incumbent from previous configs + random_starting_points = next_configs_by_random_search_sorted[: self._local_search_iterations] next_configs_by_local_search = self._local_search._maximize( previous_configs=previous_configs, n_points=self._local_search_iterations, - additional_start_points=next_configs_by_random_search_sorted, + additional_start_points=random_starting_points, ) - # Having the configurations from random search, sorted by their - # acquisition function value is important for the first few iterations - # of SMAC. As long as the random forest predicts constant value, we - # want to use only random configurations. Having them at the begging of - # the list ensures this (even after adding the configurations by local - # search, and then sorting them) - next_configs_by_acq_value = next_configs_by_random_search_sorted + next_configs_by_local_search + next_configs_by_acq_value = next_configs_by_local_search next_configs_by_acq_value.sort(reverse=True, key=lambda x: x[0]) first_five = [f"{_[0]} ({_[1].origin})" for _ in next_configs_by_acq_value[:5]] logger.debug(f"First 5 acquisition function values of selected configurations:\n{', '.join(first_five)}") return next_configs_by_acq_value - - -class LocalAndSortedPriorRandomSearch(AbstractAcquisitionMaximizer): - """Implements SMAC's default acquisition function optimization. - - This optimizer performs local search from the previous best points according to the acquisition function, uses the - acquisition function to sort randomly sampled configurations. Random configurations are interleaved by the main SMAC - code. The random configurations are retrieved from two different ConfigurationSpaces - one which uses priors - (e.g. NormalFloatHP) and is defined by the user, and one that is a uniform version of the same space, i.e. with the - priors removed. - - Parameters - ---------- - configspace : ConfigurationSpace - The original ConfigurationSpace specified by the user. - uniform_configspace : ConfigurationSpace - A version of the user-defined ConfigurationSpace where all parameters are uniform (or have their weights removed - in the case of a categorical hyperparameter). - acquisition_function : AbstractAcquisitionFunction | None, defaults to None - challengers : int, defaults to 5000 - Number of challengers. - max_steps: int, defaults to None - [LocalSearch] Maximum number of steps that the local search will perform. - n_steps_plateau_walk: int, defaults to 10 - [LocalSearch] number of steps during a plateau walk before local search terminates. - local_search_iterations: int, defaults to 10 - [Local Search] number of local search iterations. - prior_sampling_fraction: float, defaults to 0.5 - The ratio of random samples that are taken from the user-defined ConfigurationSpace, as opposed to the uniform - version. - seed : int, defaults to 0 - """ - - def __init__( - self, - configspace: ConfigurationSpace, - uniform_configspace: ConfigurationSpace, - acquisition_function: AbstractAcquisitionFunction | None = None, - challengers: int = 5000, - max_steps: int | None = None, - n_steps_plateau_walk: int = 10, - local_search_iterations: int = 10, - prior_sampling_fraction: float = 0.5, - seed: int = 0, - ) -> None: - super().__init__( - acquisition_function, - configspace, - challengers=challengers, - seed=seed, - ) - - self._prior_random_search = RandomSearch( - acquisition_function=acquisition_function, - configspace=configspace, - seed=seed, - ) - - self._uniform_random_search = RandomSearch( - acquisition_function=acquisition_function, - configspace=uniform_configspace, - seed=seed, - ) - - self._local_search = LocalSearch( - acquisition_function=acquisition_function, - configspace=configspace, - max_steps=max_steps, - n_steps_plateau_walk=n_steps_plateau_walk, - seed=seed, - ) - - self._local_search_iterations = local_search_iterations - self._prior_sampling_fraction = prior_sampling_fraction - - def _maximize( - self, - previous_configs: list[Configuration], - n_points: int, - ) -> list[tuple[float, Configuration]]: - - # Get configurations sorted by EI - next_configs_by_prior_random_search_sorted = self._prior_random_search._maximize( - previous_configs, - round(n_points * self._prior_sampling_fraction), - _sorted=True, - ) - - # Get configurations sorted by EI - next_configs_by_uniform_random_search_sorted = self._uniform_random_search._maximize( - previous_configs, - round(n_points * (1 - self._prior_sampling_fraction)), - _sorted=True, - ) - next_configs_by_random_search_sorted = [] - next_configs_by_random_search_sorted.extend(next_configs_by_prior_random_search_sorted) - next_configs_by_random_search_sorted.extend(next_configs_by_uniform_random_search_sorted) - - next_configs_by_local_search = self._local_search._maximize( - previous_configs, - self._local_search_iterations, - additional_start_points=next_configs_by_random_search_sorted, - ) - - # Having the configurations from random search, sorted by their - # acquisition function value is important for the first few iterations - # of SMAC. As long as the random forest predicts constant value, we - # want to use only random configurations. Having them at the begging of - # the list ensures this (even after adding the configurations by local - # search, and then sorting them) - next_configs_by_acq_value = next_configs_by_random_search_sorted + next_configs_by_local_search - next_configs_by_acq_value.sort(reverse=True, key=lambda x: x[0]) - logger.debug( - "First 5 acq func (origin) values of selected configurations: %s", - str([[_[0], _[1].origin] for _ in next_configs_by_acq_value[:5]]), - ) - - return next_configs_by_acq_value diff --git a/smac/acquisition/maximizer/local_search.py b/smac/acquisition/maximizer/local_search.py index c6f545f9a6..297c032a22 100644 --- a/smac/acquisition/maximizer/local_search.py +++ b/smac/acquisition/maximizer/local_search.py @@ -90,9 +90,9 @@ def _maximize( n_points: int, additional_start_points: list[tuple[float, Configuration]] | None = None, ) -> list[tuple[float, Configuration]]: - """Start a local search from the given startpoints. Iteratively collect neighbours + """Start a local search from the given start points. Iteratively collect neighbours using Configspace.utils.get_one_exchange_neighbourhood and evaluate them. - If the new config is better than the current best, the local search is coninued from the + If the new config is better than the current best, the local search is continued from the new config. Quit if either the max number of steps is reached or @@ -149,16 +149,22 @@ def _get_initial_points( list[Configuration] A list of initial points/configurations. """ - if len(previous_configs) == 0: - init_points = self._configspace.sample_configuration(size=n_points) - else: + sampled_points = [] + init_points = [] + n_init_points = n_points + if len(previous_configs) < n_points: + sampled_points = self._configspace.sample_configuration(size=n_points - len(previous_configs)) + n_init_points = len(previous_configs) + if not isinstance(sampled_points, list): + sampled_points = [sampled_points] + if len(previous_configs) > 0: init_points = self._get_init_points_from_previous_configs( previous_configs, - n_points, + n_init_points, additional_start_points, ) - return init_points + return sampled_points + init_points def _get_init_points_from_previous_configs( self, @@ -187,7 +193,7 @@ def _get_init_points_from_previous_configs( previous_configs: list[Configuration] Previous configuration (e.g., from the runhistory). n_points: int - Number of initial points to be generated. + Number of initial points to be generated; selected from previous configs (+ random configs if necessary). additional_start_points: list[tuple[float, Configuration]] | None Additional starting points. @@ -198,10 +204,6 @@ def _get_init_points_from_previous_configs( """ assert self._acquisition_function is not None - # configurations with the highest previous EI - configs_previous_runs_sorted = self._sort_by_acquisition_value(previous_configs) - configs_previous_runs_sorted = [conf[1] for conf in configs_previous_runs_sorted[:n_points]] - # configurations with the lowest predictive cost, check for None to make unit tests work if self._acquisition_function.model is not None: conf_array = convert_configurations_to_array(previous_configs) @@ -228,14 +230,13 @@ def _get_init_points_from_previous_configs( previous_configs_sorted_by_cost = [] if additional_start_points is not None: - additional_start_points = [asp[1] for asp in additional_start_points[:n_points]] + additional_start_points = [asp[1] for asp in additional_start_points] else: additional_start_points = [] init_points = [] init_points_as_set: set[Configuration] = set() for cand in itertools.chain( - configs_previous_runs_sorted, previous_configs_sorted_by_cost, additional_start_points, ): @@ -388,7 +389,7 @@ def _search( if acq_val[acq_index] > acq_val_candidates[i]: is_valid = False try: - neighbors[acq_index].is_valid_configuration() + neighbors[acq_index].check_valid_configuration() is_valid = True except (ValueError, ForbiddenValueError) as e: logger.debug("Local search %d: %s", i, e) diff --git a/smac/callback/metadata_callback.py b/smac/callback/metadata_callback.py index 626de5dee5..95a382b5f6 100644 --- a/smac/callback/metadata_callback.py +++ b/smac/callback/metadata_callback.py @@ -7,6 +7,7 @@ import smac from smac.callback.callback import Callback from smac.main.smbo import SMBO +from smac.utils.numpyencoder import NumpyEncoder __copyright__ = "Copyright 2023, AutoML.org Freiburg-Hannover" __license__ = "3-clause BSD" @@ -31,4 +32,4 @@ def on_start(self, smbo: SMBO) -> None: path.mkdir(parents=True, exist_ok=True) with open(path / "metadata.json", "w") as fp: - json.dump(meta_dict, fp, indent=2) + json.dump(meta_dict, fp, indent=2, cls=NumpyEncoder) diff --git a/smac/facade/abstract_facade.py b/smac/facade/abstract_facade.py index adb1411123..9a2031099f 100644 --- a/smac/facade/abstract_facade.py +++ b/smac/facade/abstract_facade.py @@ -92,8 +92,10 @@ class AbstractFacade: Callbacks, which are incorporated into the optimization loop. overwrite: bool, defaults to False When True, overwrites the run results if a previous run is found that is - inconsistent in the meta data with the current setup. If ``overwrite`` is set to False, the user is asked - for the exact behaviour (overwrite completely, save old run, or use old results). + consistent in the meta data with the current setup. When False and a previous run is found that is + consistent in the meta data, the run is continued. When False and a previous run is found that is + not consistent in the meta data, the the user is asked for the exact behaviour (overwrite completely + or rename old run first). dask_client: Client | None, defaults to None User-created dask client, which can be used to start a dask cluster and then attach SMAC to it. This will not be closed automatically and will have to be closed manually if provided explicitly. If none is provided @@ -115,12 +117,15 @@ def __init__( runhistory_encoder: AbstractRunHistoryEncoder | None = None, config_selector: ConfigSelector | None = None, logging_level: int | Path | Literal[False] | None = None, - callbacks: list[Callback] = [], + callbacks: list[Callback] = None, overwrite: bool = False, dask_client: Client | None = None, ): setup_logging(logging_level) + if callbacks is None: + callbacks = [] + if model is None: model = self.get_model(scenario) @@ -188,7 +193,7 @@ def __init__( # In case of multiple jobs, we need to wrap the runner again using DaskParallelRunner if (n_workers := scenario.n_workers) > 1 or dask_client is not None: - if dask_client is not None: + if dask_client is not None and n_workers > 1: logger.warning( "Provided `dask_client`. Ignore `scenario.n_workers`, directly set `n_workers` in `dask_client`." ) @@ -310,6 +315,9 @@ def optimize(self, *, data_to_scatter: dict[str, Any] | None = None) -> Configur Best found configuration. """ incumbents = None + if isinstance(data_to_scatter, dict) and len(data_to_scatter) == 0: + raise ValueError("data_to_scatter must be None or dict with some elements, but got an empty dict.") + try: incumbents = self._optimizer.optimize(data_to_scatter=data_to_scatter) finally: diff --git a/smac/facade/algorithm_configuration_facade.py b/smac/facade/algorithm_configuration_facade.py index a82e2f92ca..58229f4b72 100644 --- a/smac/facade/algorithm_configuration_facade.py +++ b/smac/facade/algorithm_configuration_facade.py @@ -125,7 +125,7 @@ def get_intensifier( def get_initial_design( # type: ignore scenario: Scenario, *, - additional_configs: list[Configuration] = [], + additional_configs: list[Configuration] = None, ) -> DefaultInitialDesign: """Returns an initial design, which returns the default configuration. @@ -134,6 +134,8 @@ def get_initial_design( # type: ignore additional_configs: list[Configuration], defaults to [] Adds additional configurations to the initial design. """ + if additional_configs is None: + additional_configs = [] return DefaultInitialDesign( scenario=scenario, additional_configs=additional_configs, diff --git a/smac/facade/blackbox_facade.py b/smac/facade/blackbox_facade.py index 9e2d4b2d1d..a5e1a18ab5 100644 --- a/smac/facade/blackbox_facade.py +++ b/smac/facade/blackbox_facade.py @@ -11,6 +11,7 @@ from smac.facade.abstract_facade import AbstractFacade from smac.initial_design.sobol_design import SobolInitialDesign from smac.intensifier.intensifier import Intensifier +from smac.main.config_selector import ConfigSelector from smac.model.gaussian_process.abstract_gaussian_process import ( AbstractGaussianProcess, ) @@ -240,7 +241,7 @@ def get_initial_design( # type: ignore n_configs: int | None = None, n_configs_per_hyperparamter: int = 8, max_ratio: float = 0.25, - additional_configs: list[Configuration] = [], + additional_configs: list[Configuration] = None, ) -> SobolInitialDesign: """Returns a Sobol design instance. @@ -259,6 +260,8 @@ def get_initial_design( # type: ignore additional_configs: list[Configuration], defaults to [] Adds additional configurations to the initial design. """ + if additional_configs is None: + additional_configs = [] return SobolInitialDesign( scenario=scenario, n_configs=n_configs, @@ -309,3 +312,15 @@ def get_runhistory_encoder( ) -> RunHistoryEncoder: """Returns the default runhistory encoder.""" return RunHistoryEncoder(scenario) + + @staticmethod + def get_config_selector( + scenario: Scenario, + *, + retrain_after: int = 1, + retries: int = 16, + ) -> ConfigSelector: + """Returns the default configuration selector.""" + return super(BlackBoxFacade, BlackBoxFacade).get_config_selector( + scenario, retrain_after=retrain_after, retries=retries + ) diff --git a/smac/facade/hyperparameter_optimization_facade.py b/smac/facade/hyperparameter_optimization_facade.py index 70a85517cd..3ed09cfdec 100644 --- a/smac/facade/hyperparameter_optimization_facade.py +++ b/smac/facade/hyperparameter_optimization_facade.py @@ -138,7 +138,7 @@ def get_initial_design( # type: ignore n_configs: int | None = None, n_configs_per_hyperparamter: int = 10, max_ratio: float = 0.25, - additional_configs: list[Configuration] = [], + additional_configs: list[Configuration] | None = None, ) -> SobolInitialDesign: """Returns a Sobol design instance. @@ -178,7 +178,7 @@ def get_random_design( # type: ignore probability : float, defaults to 0.2 Probability that a configuration will be drawn at random. """ - return ProbabilityRandomDesign(probability=probability) + return ProbabilityRandomDesign(probability=probability, seed=scenario.seed) @staticmethod def get_multi_objective_algorithm( # type: ignore diff --git a/smac/facade/multi_fidelity_facade.py b/smac/facade/multi_fidelity_facade.py index fdd52d8a4c..717162c09e 100644 --- a/smac/facade/multi_fidelity_facade.py +++ b/smac/facade/multi_fidelity_facade.py @@ -67,7 +67,7 @@ def get_initial_design( # type: ignore n_configs: int | None = None, n_configs_per_hyperparamter: int = 10, max_ratio: float = 0.25, - additional_configs: list[Configuration] = [], + additional_configs: list[Configuration] = None, ) -> RandomInitialDesign: """Returns a random initial design. @@ -80,12 +80,14 @@ def get_initial_design( # type: ignore Number of initial configurations per hyperparameter. For example, if my configuration space covers five hyperparameters and ``n_configs_per_hyperparameter`` is set to 10, then 50 initial configurations will be samples. - max_ratio: float, defaults to 0.1 + max_ratio: float, defaults to 0.25 Use at most ``scenario.n_trials`` * ``max_ratio`` number of configurations in the initial design. Additional configurations are not affected by this parameter. additional_configs: list[Configuration], defaults to [] Adds additional configurations to the initial design. """ + if additional_configs is None: + additional_configs = [] return RandomInitialDesign( scenario=scenario, n_configs=n_configs, diff --git a/smac/facade/random_facade.py b/smac/facade/random_facade.py index d5e269534c..78038ff2e3 100644 --- a/smac/facade/random_facade.py +++ b/smac/facade/random_facade.py @@ -92,7 +92,7 @@ def get_intensifier( def get_initial_design( scenario: Scenario, *, - additional_configs: list[Configuration] = [], + additional_configs: list[Configuration] = None, ) -> DefaultInitialDesign: """Returns an initial design, which returns the default configuration. @@ -101,6 +101,8 @@ def get_initial_design( additional_configs: list[Configuration], defaults to [] Adds additional configurations to the initial design. """ + if additional_configs is None: + additional_configs = [] return DefaultInitialDesign( scenario=scenario, additional_configs=additional_configs, diff --git a/smac/initial_design/abstract_initial_design.py b/smac/initial_design/abstract_initial_design.py index 7e17c88c24..d561a6772f 100644 --- a/smac/initial_design/abstract_initial_design.py +++ b/smac/initial_design/abstract_initial_design.py @@ -176,7 +176,6 @@ def _transform_continuous_designs( """ params = configspace.get_hyperparameters() for idx, param in enumerate(params): - if isinstance(param, IntegerHyperparameter): design[:, idx] = param._inverse_transform(param._transform(design[:, idx])) elif isinstance(param, NumericalHyperparameter): diff --git a/smac/intensifier/abstract_intensifier.py b/smac/intensifier/abstract_intensifier.py index 79aae61724..1acf80c8d5 100644 --- a/smac/intensifier/abstract_intensifier.py +++ b/smac/intensifier/abstract_intensifier.py @@ -26,6 +26,7 @@ from smac.scenario import Scenario from smac.utils.configspace import get_config_hash, print_config_changes from smac.utils.logging import get_logger +from smac.utils.numpyencoder import NumpyEncoder from smac.utils.pareto_front import calculate_pareto_front, sort_by_crowding_distance __copyright__ = "Copyright 2022, automl.org" @@ -571,8 +572,12 @@ def update_incumbents(self, config: Configuration) -> None: if len(previous_incumbents) == len(new_incumbents): if previous_incumbents == new_incumbents: - # No changes in the incumbents - self._remove_rejected_config(config_id) + # No changes in the incumbents, we need this clause because we can't use set difference then + if config_id in new_incumbent_ids: + self._remove_rejected_config(config_id) + else: + # config worse than incumbents and thus rejected + self._add_rejected_config(config_id) return else: # In this case, we have to determine which config replaced which incumbent and reject it @@ -662,7 +667,7 @@ def save(self, filename: str | Path) -> None: } with open(filename, "w") as fp: - json.dump(data, fp, indent=2) + json.dump(data, fp, indent=2, cls=NumpyEncoder) def get_data(self): data = { diff --git a/smac/intensifier/hyperband_utils.py b/smac/intensifier/hyperband_utils.py new file mode 100644 index 0000000000..77f6a748c6 --- /dev/null +++ b/smac/intensifier/hyperband_utils.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import numpy as np + +from smac.intensifier.successive_halving import SuccessiveHalving + + +def determine_HB(min_budget: float, max_budget: float, eta: int = 3) -> dict: + """Determine one Hyperband round + + Parameters + ---------- + min_budget : float + Minimum budget per trial in fidelity units + max_budget : float + Maximum budget per trial in fidelity units + eta : int, defaults to 3 + Input that controls the proportion of configurations discarded in each round of Successive Halving. + + Returns + ------- + dict + Info about the Hyperband round + "max_iterations" + "n_configs_in_stage" + "budgets_in_stage" + "trials_used" + "budget_used" + "number_of_brackets" + + """ + _s_max = SuccessiveHalving._get_max_iterations(eta, max_budget, min_budget) + + _max_iterations: dict[int, int] = {} + _n_configs_in_stage: dict[int, list] = {} + _budgets_in_stage: dict[int, list] = {} + + for i in range(_s_max + 1): + max_iter = _s_max - i + + _budgets_in_stage[i], _n_configs_in_stage[i] = SuccessiveHalving._compute_configs_and_budgets_for_stages( + eta, max_budget, max_iter, _s_max + ) + _max_iterations[i] = max_iter + 1 + + total_trials = np.sum([np.sum(v) for v in _n_configs_in_stage.values()]) + total_budget = np.sum([np.sum(v) for v in _budgets_in_stage.values()]) + + return { + "max_iterations": _max_iterations, + "n_configs_in_stage": _n_configs_in_stage, + "budgets_in_stage": _budgets_in_stage, + "trials_used": total_trials, + "budget_used": total_budget, + "number_of_brackets": len(_max_iterations), + } + + +def determine_hyperband_for_multifidelity( + total_budget: float, min_budget: float, max_budget: float, eta: int = 3 +) -> dict: + """Determine how many Hyperband rounds should happen based on a total budget + + Parameters + ---------- + total_budget : float + Total budget for the complete optimization in fidelity units + min_budget : float + Minimum budget per trial in fidelity units + max_budget : float + Maximum budget per trial in fidelity units + eta : int, defaults to 3 + Input that controls the proportion of configurations discarded in each round of Successive Halving. + + Returns + ------- + dict + Info about one Hyperband round + "max_iterations" + "n_configs_in_stage" + "budgets_in_stage" + "trials_used" + "budget_used" + "number_of_brackets" + Info about whole optimization + "n_trials" + "total_budget" + "eta" + "min_budget" + "max_budget" + + """ + # Determine the HB + hyperband_round = determine_HB(eta=eta, min_budget=min_budget, max_budget=max_budget) + + # Calculate how many HB rounds we can have + budget_used_per_hyperband_round = hyperband_round["budget_used"] + number_of_full_hb_rounds = int(np.floor(total_budget / budget_used_per_hyperband_round)) + remaining_budget = total_budget % budget_used_per_hyperband_round + trials_used_per_hb_round = hyperband_round["trials_used"] + n_configs_in_stage = hyperband_round["n_configs_in_stage"] + budgets_in_stage = hyperband_round["budgets_in_stage"] + + remaining_trials = 0 + for stage in n_configs_in_stage.keys(): + B = budgets_in_stage[stage] + C = n_configs_in_stage[stage] + for b, c in zip(B, C): + # How many trials are left? + # If b * c is lower than remaining budget, we can add full c + # otherwise we need to find out how many trials we can do with this budget + remaining_trials += min(c, int(np.floor(remaining_budget / b))) + # We cannot go lower than 0 + # If we are in the case of b*c > remaining_budget, we will not have any + # budget left. We can not add full c but the number of trials that still fit + remaining_budget = max(0, remaining_budget - b * c) + + n_trials = int(number_of_full_hb_rounds * trials_used_per_hb_round + remaining_trials) + + hyperband_info = hyperband_round + hyperband_info["n_trials"] = n_trials + hyperband_info["total_budget"] = total_budget + hyperband_info["eta"] = eta + hyperband_info["min_budget"] = min_budget + hyperband_info["max_budget"] = max_budget + + return hyperband_info + + +def print_hyperband_summary(hyperband_info: dict) -> None: + """Print summary about Hyperband as used in the MultiFidelityFacade + + Parameters + ---------- + hyperband_info : dict + Info dict about Hyperband + """ + print("-" * 30, "HYPERBAND IN MULTI-FIDELITY", "-" * 30) + print("total budget:\t\t", hyperband_info["total_budget"]) + print("total number of trials:\t", hyperband_info["n_trials"]) + print("number of HB rounds:\t", hyperband_info["total_budget"] / hyperband_info["budget_used"]) + print() + + print("\t~~~~~~~~~~~~HYPERBAND ROUND") + print("\teta:\t\t\t\t\t", hyperband_info["eta"]) + print("\tmin budget per trial:\t\t\t", hyperband_info["min_budget"]) + print("\tmax budget per trial:\t\t\t", hyperband_info["max_budget"]) + print("\ttotal number of trials per HB round:\t", hyperband_info["trials_used"]) + print("\tbudget used per HB round:\t\t", hyperband_info["budget_used"]) + print("\tnumber of brackets:\t\t\t", hyperband_info["number_of_brackets"]) + print("\tbudgets per stage:\t\t\t", hyperband_info["budgets_in_stage"]) + print("\tn configs per stage:\t\t\t", hyperband_info["n_configs_in_stage"]) + print("-" * (2 * 30 + len("HYPERBAND IN MULTI-FIDELITY") + 2)) + + +def get_n_trials_for_hyperband_multifidelity( + total_budget: float, min_budget: float, max_budget: float, eta: int = 3, print_summary: bool = True +) -> int: + """Calculate the number of trials needed for multi-fidelity optimization + + Specify the total budget and find out how many trials that equals. + + Parameters + ---------- + total_budget : float + Total budget for the complete optimization in fidelity units. + A fidelity unit can be one epoch or a fraction of a dataset size. + min_budget : float + Minimum budget per trial in fidelity units + max_budget : float + Maximum budget per trial in fidelity units + eta : int, defaults to 3 + Input that controls the proportion of configurations discarded in each round of Successive Halving. + + Returns + ------- + int + Number of trials needed for the specified total budgets + """ + hyperband_info = determine_hyperband_for_multifidelity( + total_budget=total_budget, eta=eta, min_budget=min_budget, max_budget=max_budget + ) + if print_summary: + print_hyperband_summary(hyperband_info=hyperband_info) + return hyperband_info["n_trials"] diff --git a/smac/intensifier/successive_halving.py b/smac/intensifier/successive_halving.py index 01b95c6d86..58960ca67e 100644 --- a/smac/intensifier/successive_halving.py +++ b/smac/intensifier/successive_halving.py @@ -384,7 +384,7 @@ def __iter__(self) -> Iterator[TrialInfo]: # noqa: D102 logger.debug("Updating tracker:") # TODO: Process stages ascending or descending? - for (bracket, stage) in list(self._tracker.keys()): + for bracket, stage in list(self._tracker.keys()): pairs = self._tracker[(bracket, stage)].copy() for seed, configs in pairs: isb_keys = self._get_instance_seed_budget_keys_by_stage(bracket=bracket, stage=stage, seed=seed) diff --git a/smac/main/config_selector.py b/smac/main/config_selector.py index 76ace17f84..4e3574d589 100644 --- a/smac/main/config_selector.py +++ b/smac/main/config_selector.py @@ -91,7 +91,7 @@ def _set_components( acquisition_maximizer: AbstractAcquisitionMaximizer, acquisition_function: AbstractAcquisitionFunction, random_design: AbstractRandomDesign, - callbacks: list[Callback] = [], + callbacks: list[Callback] = None, ) -> None: self._runhistory = runhistory self._runhistory_encoder = runhistory_encoder @@ -99,7 +99,7 @@ def _set_components( self._acquisition_maximizer = acquisition_maximizer self._acquisition_function = acquisition_function self._random_design = random_design - self._callbacks = callbacks + self._callbacks = callbacks if callbacks is not None else [] self._initial_design_configs = initial_design.select_configurations() if len(self._initial_design_configs) == 0: @@ -213,7 +213,6 @@ def __iter__(self) -> Iterator[Configuration]: # Now we maximize the acquisition function challengers = self._acquisition_maximizer.maximize( previous_configs, - n_points=self._retrain_after, random_design=self._random_design, ) @@ -266,14 +265,14 @@ def _collect_data(self) -> tuple[np.ndarray, np.ndarray, np.ndarray]: assert self._runhistory_encoder is not None # If we use a float value as a budget, we want to train the model only on the highest budget - available_budgets = [] - for run_key in self._runhistory: - budget = run_key.budget - if budget not in available_budgets: - available_budgets.append(run_key.budget) + unique_budgets: set[float] = {run_key.budget for run_key in self._runhistory if run_key.budget is not None} - # Sort available budgets from highest to lowest budget - available_budgets = sorted(list(set(available_budgets)), reverse=True) # type: ignore + available_budgets: list[float] | list[None] + if len(unique_budgets) > 0: + # Sort available budgets from highest to lowest budget + available_budgets = sorted(unique_budgets, reverse=True) + else: + available_budgets = [None] # Get #points per budget and if there are enough samples, then build a model for b in available_budgets: @@ -282,7 +281,7 @@ def _collect_data(self) -> tuple[np.ndarray, np.ndarray, np.ndarray]: if X.shape[0] >= self._min_trials: self._considered_budgets = [b] - # TODO: Add running configs + # Possible add running configs? configs_array = self._runhistory_encoder.get_configurations(budget_subset=self._considered_budgets) return X, Y, configs_array diff --git a/smac/main/smbo.py b/smac/main/smbo.py index 30e01c2e97..dd6bfd3548 100644 --- a/smac/main/smbo.py +++ b/smac/main/smbo.py @@ -24,6 +24,7 @@ from smac.scenario import Scenario from smac.utils.data_structures import recursively_compare_dicts from smac.utils.logging import get_logger +from smac.utils.numpyencoder import NumpyEncoder __copyright__ = "Copyright 2022, automl.org" __license__ = "3-clause BSD" @@ -414,7 +415,7 @@ def save(self) -> None: # Save optimization data with open(str(path / "optimization.json"), "w") as file: - json.dump(data, file, indent=2) + json.dump(data, file, indent=2, cls=NumpyEncoder) # And save runhistory and intensifier self._runhistory.save(path / "runhistory.json") @@ -461,18 +462,22 @@ def _add_results(self) -> None: logger.info("Cost threshold was reached. Abort is requested.") self._stop = True - def register_callback(self, callback: Callback, index: int = -1) -> None: + def register_callback(self, callback: Callback, index: int | None = None) -> None: """ Registers a callback to be called before, in between, and after the Bayesian optimization loop. + Callback is appended to the list by default. Parameters ---------- callback : Callback The callback to be registered. - index : int - The index at which the callback should be registered. + index : int, optional + The index at which the callback should be registered. The default is None. + If it is None, append the callback to the list. """ + if index is None: + index = len(self._callbacks) self._callbacks.insert(index, callback) def _initialize_state(self) -> None: @@ -501,7 +506,7 @@ def _initialize_state(self) -> None: logger.info("Since the previous run was not successful, SMAC will start from scratch again.") self.reset() else: - # Here, we run into differen scenarios + # Here, we run into different scenarios diff = recursively_compare_dicts( Scenario.make_serializable(self._scenario), Scenario.make_serializable(old_scenario), diff --git a/smac/model/gaussian_process/kernels/base_kernels.py b/smac/model/gaussian_process/kernels/base_kernels.py index 044ad4fcaa..4dd6ff6bbf 100644 --- a/smac/model/gaussian_process/kernels/base_kernels.py +++ b/smac/model/gaussian_process/kernels/base_kernels.py @@ -250,12 +250,12 @@ def _signature(self, func: Callable) -> Signature: def _set_active_dims(self, operate_on: np.ndarray | None = None) -> None: """Sets dimensions this kernel should work on.""" - if operate_on is not None and type(operate_on) in (list, np.ndarray): + if operate_on is not None and isinstance(operate_on, (list, np.ndarray)): if not isinstance(operate_on, np.ndarray): - raise TypeError("The argument `operate_on` needs to be of type np.ndarray but is %s" % type(operate_on)) + raise TypeError(f"The argument `operate_on` needs to be of type np.ndarray but is {type(operate_on)}") - if operate_on.dtype != int: - raise ValueError("The dtype of argument `operate_on` needs to be int, but is %s" % operate_on.dtype) + if not np.issubdtype(operate_on.dtype, np.integer): + raise ValueError(f"The dtype of `operate_on` needs to be np.integer, but is {operate_on.dtype}") self.operate_on = operate_on self._len_active = len(operate_on) diff --git a/smac/model/gaussian_process/kernels/rbf_kernel.py b/smac/model/gaussian_process/kernels/rbf_kernel.py index 5bf2076588..13aaac49f1 100644 --- a/smac/model/gaussian_process/kernels/rbf_kernel.py +++ b/smac/model/gaussian_process/kernels/rbf_kernel.py @@ -24,7 +24,6 @@ def __init__( has_conditions: bool = False, prior: AbstractPrior | None = None, ) -> None: - super().__init__( operate_on=operate_on, has_conditions=has_conditions, diff --git a/smac/model/gaussian_process/kernels/white_kernel.py b/smac/model/gaussian_process/kernels/white_kernel.py index a3fa4a61b3..f8a7814a77 100644 --- a/smac/model/gaussian_process/kernels/white_kernel.py +++ b/smac/model/gaussian_process/kernels/white_kernel.py @@ -21,7 +21,6 @@ def __init__( has_conditions: bool = False, prior: AbstractPrior | None = None, ) -> None: - super().__init__( operate_on=operate_on, has_conditions=has_conditions, diff --git a/smac/model/gaussian_process/mcmc_gaussian_process.py b/smac/model/gaussian_process/mcmc_gaussian_process.py index 7c4ff40287..d6a098159c 100644 --- a/smac/model/gaussian_process/mcmc_gaussian_process.py +++ b/smac/model/gaussian_process/mcmc_gaussian_process.py @@ -247,7 +247,6 @@ def _train( assert self._samples is not None for sample in self._samples: - if (sample < -50).any(): sample[sample < -50] = -50 if (sample > 50).any(): diff --git a/smac/model/random_forest/abstract_random_forest.py b/smac/model/random_forest/abstract_random_forest.py index d4f1f7ce30..e407331be5 100644 --- a/smac/model/random_forest/abstract_random_forest.py +++ b/smac/model/random_forest/abstract_random_forest.py @@ -6,6 +6,7 @@ from ConfigSpace import ( CategoricalHyperparameter, Constant, + OrdinalHyperparameter, UniformFloatHyperparameter, UniformIntegerHyperparameter, ) @@ -36,12 +37,14 @@ def _impute_inactive(self, X: np.ndarray) -> np.ndarray: self._conditional[idx] = True if isinstance(hp, CategoricalHyperparameter): self._impute_values[idx] = len(hp.choices) + elif isinstance(hp, OrdinalHyperparameter): + self._impute_values[idx] = len(hp.sequence) elif isinstance(hp, (UniformFloatHyperparameter, UniformIntegerHyperparameter)): self._impute_values[idx] = -1 elif isinstance(hp, Constant): self._impute_values[idx] = 1 else: - raise ValueError + raise ValueError(f"Unsupported hyperparameter type: {type(hp)}") if self._conditional[idx] is True: nonfinite_mask = ~np.isfinite(X[:, idx]) diff --git a/smac/model/random_forest/random_forest.py b/smac/model/random_forest/random_forest.py index 6d7840d2ac..72685803f9 100644 --- a/smac/model/random_forest/random_forest.py +++ b/smac/model/random_forest/random_forest.py @@ -87,7 +87,9 @@ def __init__( self._rf_opts.compute_law_of_total_variance = False self._rf: BinaryForest | None = None self._log_y = log_y - self._rng = regression.default_random_engine(seed) + + # Case to `int` incase we get an `np.integer` type + self._rng = regression.default_random_engine(int(seed)) self._n_trees = n_trees self._n_points_per_tree = n_points_per_tree @@ -211,7 +213,7 @@ def _predict( third_dimension = max(max_num_leaf_data, third_dimension) # Transform list of 2d arrays into a 3d array - preds_as_array = np.zeros((X.shape[0], self._rf_opts.num_trees, third_dimension)) * np.NaN + preds_as_array = np.zeros((X.shape[0], self._rf_opts.num_trees, third_dimension)) * np.nan for i, preds_per_tree in enumerate(all_preds): for j, pred in enumerate(preds_per_tree): preds_as_array[i, j, : len(pred)] = pred diff --git a/smac/runhistory/encoder/abstract_encoder.py b/smac/runhistory/encoder/abstract_encoder.py index 42454771ed..e9de8b14cb 100644 --- a/smac/runhistory/encoder/abstract_encoder.py +++ b/smac/runhistory/encoder/abstract_encoder.py @@ -42,17 +42,17 @@ class AbstractRunHistoryEncoder: def __init__( self, scenario: Scenario, - considered_states: list[StatusType] = [ - StatusType.SUCCESS, - StatusType.CRASHED, - StatusType.MEMORYOUT, - ], - lower_budget_states: list[StatusType] = [], + considered_states: list[StatusType] = None, + lower_budget_states: list[StatusType] = None, scale_percentage: int = 5, seed: int | None = None, ) -> None: if considered_states is None: - raise TypeError("No success states are given.") + considered_states = [ + StatusType.SUCCESS, + StatusType.CRASHED, + StatusType.MEMORYOUT, + ] if seed is None: seed = scenario.seed @@ -62,7 +62,7 @@ def __init__( self._scale_percentage = scale_percentage self._n_objectives = scenario.count_objectives() self._algorithm_walltime_limit = scenario.trial_walltime_limit - self._lower_budget_states = lower_budget_states + self._lower_budget_states = lower_budget_states if lower_budget_states is not None else [] self._considered_states = considered_states self._instances = scenario.instances @@ -80,9 +80,9 @@ def __init__( ) # Learned statistics - self._min_y = np.array([np.NaN] * self._n_objectives) - self._max_y = np.array([np.NaN] * self._n_objectives) - self._percentile = np.array([np.NaN] * self._n_objectives) + self._min_y = np.array([np.nan] * self._n_objectives) + self._max_y = np.array([np.nan] * self._n_objectives) + self._percentile = np.array([np.nan] * self._n_objectives) self._multi_objective_algorithm: AbstractMultiObjectiveAlgorithm | None = None self._runhistory: RunHistory | None = None diff --git a/smac/runhistory/runhistory.py b/smac/runhistory/runhistory.py index a30e1356ee..091e3f95b2 100644 --- a/smac/runhistory/runhistory.py +++ b/smac/runhistory/runhistory.py @@ -24,6 +24,7 @@ from smac.utils.configspace import get_config_hash from smac.utils.logging import get_logger from smac.utils.multi_objective import normalize_costs +from smac.utils.numpyencoder import NumpyEncoder __copyright__ = "Copyright 2022, automl.org" __license__ = "3-clause BSD" @@ -178,7 +179,7 @@ def add( budget: float | None = None, starttime: float = 0.0, endtime: float = 0.0, - additional_info: dict[str, Any] = {}, + additional_info: dict[str, Any] = None, force_update: bool = False, ) -> None: """Adds a new trial to the RunHistory. @@ -205,6 +206,8 @@ def add( raise TypeError("Configuration must not be None.") elif not isinstance(config, Configuration): raise TypeError("Configuration is not of type Configuration, but %s." % type(config)) + if additional_info is None: + additional_info = {} # Squeeze is important to reduce arrays with one element # to scalars. @@ -435,7 +438,7 @@ def get_min_cost(self, config: Configuration) -> float: cost = self._min_cost_per_config.get(config_id, np.nan) # type: ignore if self._n_objectives > 1: - assert type(cost) == list + assert isinstance(cost, list) assert self.multi_objective_algorithm is not None costs = normalize_costs(cost, self._objective_bounds) @@ -443,7 +446,7 @@ def get_min_cost(self, config: Configuration) -> float: # Note: We have to mean here because we already got the min cost return self.multi_objective_algorithm(costs) - assert type(cost) == float + assert isinstance(cost, float) return float(cost) def average_cost( @@ -801,6 +804,7 @@ def save(self, filename: str | Path = "runhistory.json") -> None: }, fp, indent=2, + cls=NumpyEncoder, ) def load(self, filename: str | Path, configspace: ConfigurationSpace) -> None: @@ -847,7 +851,7 @@ def load(self, filename: str | Path, configspace: ConfigurationSpace) -> None: for entry in data["data"]: # Set n_objectives first if self._n_objectives == -1: - if isinstance(entry[4], float) or isinstance(entry[4], int): + if isinstance(entry[4], (float, int)): self._n_objectives = 1 else: self._n_objectives = len(entry[4]) @@ -953,7 +957,7 @@ def _check_json_serializable( trial_value: TrialValue, ) -> None: try: - json.dumps(obj) + json.dumps(obj, cls=NumpyEncoder) except Exception as e: raise ValueError( "Cannot add %s: %s of type %s to runhistory because it raises an error during JSON encoding, " diff --git a/smac/runner/abstract_runner.py b/smac/runner/abstract_runner.py index 2631bd966d..4e15cf4b5f 100644 --- a/smac/runner/abstract_runner.py +++ b/smac/runner/abstract_runner.py @@ -50,8 +50,10 @@ class AbstractRunner(ABC): def __init__( self, scenario: Scenario, - required_arguments: list[str] = [], + required_arguments: list[str] = None, ): + if required_arguments is None: + required_arguments = [] self._scenario = scenario self._required_arguments = required_arguments diff --git a/smac/runner/dask_runner.py b/smac/runner/dask_runner.py index b9aade4015..d4bb528bdc 100644 --- a/smac/runner/dask_runner.py +++ b/smac/runner/dask_runner.py @@ -91,7 +91,7 @@ def __init__( ) if self._scenario.output_directory is not None: - self._scheduler_file = self._scenario.output_directory / ".dask_scheduler_file" + self._scheduler_file = Path(self._scenario.output_directory, ".dask_scheduler_file") self._client.write_scheduler_file(scheduler_file=str(self._scheduler_file)) else: # We just use their set up diff --git a/smac/runner/target_function_runner.py b/smac/runner/target_function_runner.py index 28837caa09..8a14c2fc9e 100644 --- a/smac/runner/target_function_runner.py +++ b/smac/runner/target_function_runner.py @@ -7,6 +7,7 @@ import math import time import traceback +from functools import partial import numpy as np from ConfigSpace import Configuration @@ -44,8 +45,10 @@ def __init__( self, scenario: Scenario, target_function: Callable, - required_arguments: list[str] = [], + required_arguments: list[str] = None, ): + if required_arguments is None: + required_arguments = [] super().__init__(scenario=scenario, required_arguments=required_arguments) self._target_function = target_function @@ -88,7 +91,17 @@ def __init__( @property def meta(self) -> dict[str, Any]: # noqa: D102 meta = super().meta - meta.update({"code": str(self._target_function.__code__.co_code)}) + + # Partial's don't have a __code__ attribute but are a convenient + # way a user might want to pass a function to SMAC, specifying + # keyword arguments. + f = self._target_function + if isinstance(f, partial): + f = f.func + meta.update({"code": str(f.__code__.co_code)}) + meta.update({"code-partial-args": repr(f)}) + else: + meta.update({"code": str(self._target_function.__code__.co_code)}) return meta diff --git a/smac/runner/target_function_script_runner.py b/smac/runner/target_function_script_runner.py index 17feffc983..5ab558906d 100644 --- a/smac/runner/target_function_script_runner.py +++ b/smac/runner/target_function_script_runner.py @@ -51,8 +51,10 @@ def __init__( self, target_function: str, scenario: Scenario, - required_arguments: list[str] = [], + required_arguments: list[str] = None, ): + if required_arguments is None: + required_arguments = [] super().__init__(scenario=scenario, required_arguments=required_arguments) self._target_function = target_function diff --git a/smac/scenario.py b/smac/scenario.py index 133ae57b07..ca0df81a20 100644 --- a/smac/scenario.py +++ b/smac/scenario.py @@ -11,9 +11,9 @@ import numpy as np from ConfigSpace import ConfigurationSpace -from ConfigSpace.read_and_write import json as cs_json from smac.utils.logging import get_logger +from smac.utils.numpyencoder import NumpyEncoder logger = get_logger(__name__) @@ -203,12 +203,11 @@ def save(self) -> None: # Save everything filename = self.output_directory / "scenario.json" with open(filename, "w") as fh: - json.dump(data, fh, indent=4) + json.dump(data, fh, indent=4, cls=NumpyEncoder) # Save configspace on its own configspace_filename = self.output_directory / "configspace.json" - with open(configspace_filename, "w") as f: - f.write(cs_json.write(self.configspace)) + self.configspace.to_json(configspace_filename) @staticmethod def load(path: Path) -> Scenario: @@ -224,9 +223,7 @@ def load(path: Path) -> Scenario: # Read configspace configspace_filename = path / "configspace.json" - with open(configspace_filename, "r") as f: - - configspace = cs_json.read(f.read()) + configspace = ConfigurationSpace.from_json(configspace_filename) data["configspace"] = configspace diff --git a/smac/utils/configspace.py b/smac/utils/configspace.py index e6fdeddbdf..8224f3ef90 100644 --- a/smac/utils/configspace.py +++ b/smac/utils/configspace.py @@ -101,22 +101,22 @@ def get_types( if can_be_inactive: raise ValueError("Inactive parameters not supported for Beta and Normal Hyperparameters") - bounds[i] = (param._lower, param._upper) + bounds[i] = (param.lower_vectorized, param.upper_vectorized) elif isinstance(param, NormalIntegerHyperparameter): if can_be_inactive: raise ValueError("Inactive parameters not supported for Beta and Normal Hyperparameters") - bounds[i] = (param.nfhp._lower, param.nfhp._upper) + bounds[i] = (param.lower_vectorized, param.upper_vectorized) elif isinstance(param, BetaFloatHyperparameter): if can_be_inactive: raise ValueError("Inactive parameters not supported for Beta and Normal Hyperparameters") - bounds[i] = (param._lower, param._upper) + bounds[i] = (param.lower_vectorized, param.upper_vectorized) elif isinstance(param, BetaIntegerHyperparameter): if can_be_inactive: raise ValueError("Inactive parameters not supported for Beta and Normal Hyperparameters") - bounds[i] = (param.bfhp._lower, param.bfhp._upper) + bounds[i] = (param.lower_vectorized, param.upper_vectorized) elif not isinstance( param, ( @@ -169,12 +169,17 @@ def print_config_changes( if incumbent is None or challenger is None: return - params = sorted([(param, incumbent[param], challenger[param]) for param in challenger.keys()]) - for param in params: - if param[1] != param[2]: - logger.info("--- %s: %r -> %r" % param) - else: - logger.debug("--- %s Remains unchanged: %r", param[0], param[1]) + inc_keys = set(incumbent.keys()) + all_keys = inc_keys.union(challenger.keys()) + + lines = [] + for k in sorted(all_keys): + inc_k = incumbent.get(k, "-inactive-") + cha_k = challenger.get(k, "-inactive-") + lines.append(f"--- {k}: {inc_k} -> {cha_k}" + " (unchanged)" if inc_k == cha_k else "") + + msg = "\n".join(lines) + logger.debug(msg) # def check_subspace_points( diff --git a/smac/utils/multi_objective.py b/smac/utils/multi_objective.py index 2dec4da949..f959fe7836 100644 --- a/smac/utils/multi_objective.py +++ b/smac/utils/multi_objective.py @@ -29,7 +29,7 @@ def normalize_costs( costs = [] for v, b in zip(values, bounds): - assert type(v) != list + assert not isinstance(v, list) p = v - b[0] q = b[1] - b[0] diff --git a/smac/utils/numpyencoder.py b/smac/utils/numpyencoder.py new file mode 100644 index 0000000000..c7b2b6c7b4 --- /dev/null +++ b/smac/utils/numpyencoder.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from typing import Any + +import json + +import numpy as np + + +class NumpyEncoder(json.JSONEncoder): + """Custom encoder for numpy data types + + From https://stackoverflow.com/a/61903895 + """ + + def default(self, obj: Any) -> Any: + """Handle numpy datatypes if present by converting to native python + + Parameters + ---------- + obj : Any + Object to serialize + + Returns + ------- + Any + Object in native python + """ + if isinstance( + obj, + ( + np.int_, + np.intc, + np.intp, + np.int8, + np.int16, + np.int32, + np.int64, + np.uint8, + np.uint16, + np.uint32, + np.uint64, + ), + ): + return int(obj) + + elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)): + return float(obj) + + elif isinstance(obj, (np.complex_, np.complex64, np.complex128)): + return {"real": obj.real, "imag": obj.imag} + + elif isinstance(obj, (np.ndarray,)): + return obj.tolist() + + elif isinstance(obj, (np.bool_)): + return bool(obj) + + elif isinstance(obj, (np.void)): + return None + + return json.JSONEncoder.default(self, obj) diff --git a/smac/utils/subspaces/__init__.py b/smac/utils/subspaces/__init__.py index 686099830f..77d5b97f5a 100644 --- a/smac/utils/subspaces/__init__.py +++ b/smac/utils/subspaces/__init__.py @@ -322,7 +322,7 @@ # hp_list.append(hp_new) # # We only consider plain hyperparameters -# self.cs_local.add_hyperparameters(hp_list) +# self.cs_local.add(hp_list) # forbiddens_ss = [] # forbiddens = config_space.get_forbiddens() # for forbidden in forbiddens: diff --git a/tests/conftest.py b/tests/conftest.py index 92a415162e..5b26d6b708 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -143,7 +143,6 @@ def pytest_sessionfinish(session: Session, exitstatus: ExitCode) -> None: proc = psutil.Process() kill_signal = signal.SIGTERM for child in proc.children(recursive=True): - # https://stackoverflow.com/questions/57336095/access-verbosity-level-in-a-pytest-helper-function if session.config.getoption("verbose") > 0: print(child, child.cmdline()) diff --git a/tests/fixtures/configspace.py b/tests/fixtures/configspace.py index 77cf9522d3..5e6e0ca3ff 100644 --- a/tests/fixtures/configspace.py +++ b/tests/fixtures/configspace.py @@ -18,7 +18,7 @@ def configspace_small() -> ConfigurationSpace: c = Categorical("c", ["cat", "dog", "mouse"], default="cat") # Add all hyperparameters at once: - cs.add_hyperparameters([a, b, c]) + cs.add([a, b, c]) return cs @@ -36,7 +36,7 @@ def configspace_large() -> ConfigurationSpace: learning_rate_init = Float("learning_rate_init", (0.0001, 1.0), default=0.001, log=True) # Add all hyperparameters at once: - cs.add_hyperparameters( + cs.add( [ n_layer, n_neurons, @@ -57,6 +57,6 @@ def configspace_large() -> ConfigurationSpace: use_batch_size = InCondition(child=batch_size, parent=solver, values=["sgd", "adam"]) # We can also add multiple conditions on hyperparameters at once: - cs.add_conditions([use_lr, use_batch_size, use_lr_init]) + cs.add([use_lr, use_batch_size, use_lr_init]) return cs diff --git a/tests/fixtures/models.py b/tests/fixtures/models.py index 1321176f27..7a7a8fcc09 100644 --- a/tests/fixtures/models.py +++ b/tests/fixtures/models.py @@ -18,7 +18,7 @@ def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) x0 = Float("x0", (-5, 10), default=-3) x1 = Float("x1", (-5, 10), default=-4) - cs.add_hyperparameters([x0, x1]) + cs.add([x0, x1]) return cs @@ -58,7 +58,7 @@ def configspace(self) -> ConfigurationSpace: eta0 = Float("eta0", (0.00001, 1), default=0.1, log=True) # Add the parameters to configuration space - cs.add_hyperparameters([alpha, l1_ratio, learning_rate, eta0]) + cs.add([alpha, l1_ratio, learning_rate, eta0]) return cs @@ -71,7 +71,7 @@ def train(self, config: Configuration, instance: str = "0-1", budget: float = 1, # SGD classifier using given configuration clf = SGDClassifier( - loss="log", + loss="log_loss", penalty="elasticnet", alpha=config["alpha"], l1_ratio=config["l1_ratio"], diff --git a/tests/test_acquisition/test_maximizers.py b/tests/test_acquisition/test_maximizers.py index 4cd4110f34..d7698e0a29 100644 --- a/tests/test_acquisition/test_maximizers.py +++ b/tests/test_acquisition/test_maximizers.py @@ -30,7 +30,6 @@ from smac.acquisition.function import EI from smac.acquisition.maximizer import ( DifferentialEvolution, - LocalAndSortedPriorRandomSearch, LocalAndSortedRandomSearch, LocalSearch, RandomSearch, @@ -55,8 +54,8 @@ def get_array(self): def configspace_branin() -> ConfigurationSpace: """Returns the branin configspace.""" cs = ConfigurationSpace() - cs.add_hyperparameter(Float("x", (-5, 10))) - cs.add_hyperparameter(Float("y", (0, 15))) + cs.add(Float("x", (-5, 10))) + cs.add(Float("y", (0, 15))) return cs @@ -196,7 +195,7 @@ def configspace() -> ConfigurationSpace: c = Float("c", (0, 1), default=0.5) # Add all hyperparameters at once: - cs.add_hyperparameters([a, b, c]) + cs.add([a, b, c]) return cs @@ -263,7 +262,6 @@ def predict_marginalized(self, X): return X, X class AcquisitionFunction: - model = Model() def __call__(self, X): @@ -278,7 +276,7 @@ def __call__(self, X): random_configs = configspace.sample_configuration(size=100) points = ls._get_initial_points(random_configs, n_points=5, additional_start_points=None) - assert len(points) == 10 + assert len(points) == 5 # -------------------------------------------------------------- @@ -334,7 +332,7 @@ def test_local_and_random_search(configspace, acquisition_function): values = rs._maximize(start_points, 100) config_origins = [] v_old = np.inf - for (v, config) in values: + for v, config in values: config_origins += [config.origin] if isinstance(v, np.ndarray): v = float(v[0]) @@ -342,7 +340,6 @@ def test_local_and_random_search(configspace, acquisition_function): assert v_old >= v v_old = v - assert "Acquisition Function Maximizer: Random Search (sorted)" in config_origins assert "Acquisition Function Maximizer: Local Search" in config_origins @@ -359,7 +356,7 @@ def configspace_rosenbrock(): x2 = UniformIntegerHyperparameter("x2", -5, 5, default_value=5) x3 = CategoricalHyperparameter("x3", [5, 2, 0, 1, -1, -2, 4, -3, 3, -5, -4], default_value=5) x4 = UniformIntegerHyperparameter("x4", -5, 5, default_value=5) - uniform_cs.add_hyperparameters([x1, x2, x3, x4]) + uniform_cs.add([x1, x2, x3, x4]) return uniform_cs @@ -375,7 +372,7 @@ def configspace_prior(): "x3", [5, 2, 0, 1, -1, -2, 4, -3, 3, -5, -4], default_value=5, weights=[999, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] ) x4 = UniformIntegerHyperparameter("x4", lower=-5, upper=5, default_value=5) - prior_cs.add_hyperparameters([x1, x2, x3, x4]) + prior_cs.add([x1, x2, x3, x4]) return prior_cs @@ -390,26 +387,26 @@ def __call__(self, arrays): budget_kwargs = {"max_steps": 2, "n_steps_plateau_walk": 2, "local_search_iterations": 2} - prs_0 = LocalAndSortedPriorRandomSearch( - configspace_prior, - configspace_rosenbrock, - AcquisitionFunction(), + prs_0 = LocalAndSortedRandomSearch( + configspace=configspace_prior, + uniform_configspace=configspace_rosenbrock, + acquisition_function=AcquisitionFunction(), prior_sampling_fraction=0, **budget_kwargs, ) - prs_05 = LocalAndSortedPriorRandomSearch( - configspace_prior, - configspace_rosenbrock, - AcquisitionFunction(), + prs_05 = LocalAndSortedRandomSearch( + configspace=configspace_prior, + uniform_configspace=configspace_rosenbrock, + acquisition_function=AcquisitionFunction(), prior_sampling_fraction=0.9, **budget_kwargs, ) - prs_1 = LocalAndSortedPriorRandomSearch( - configspace_prior, - configspace_rosenbrock, - AcquisitionFunction(), + prs_1 = LocalAndSortedRandomSearch( + configspace=configspace_prior, + uniform_configspace=configspace_rosenbrock, + acquisition_function=AcquisitionFunction(), prior_sampling_fraction=1, **budget_kwargs, ) diff --git a/tests/test_initial_design/test_sobol_design.py b/tests/test_initial_design/test_sobol_design.py index 8663e2972f..8fbb427e8f 100644 --- a/tests/test_initial_design/test_sobol_design.py +++ b/tests/test_initial_design/test_sobol_design.py @@ -24,7 +24,7 @@ def test_sobol_design(make_scenario, configspace_large): def test_max_hyperparameters(make_scenario): cs = ConfigurationSpace() hyperparameters = [Float("x%d" % (i + 1), (0, 1)) for i in range(21202)] - cs.add_hyperparameters(hyperparameters) + cs.add(hyperparameters) scenario = make_scenario(cs) diff --git a/tests/test_intensifier/test_abstract_intensifier.py b/tests/test_intensifier/test_abstract_intensifier.py index ce980c49b9..6d878f9154 100644 --- a/tests/test_intensifier/test_abstract_intensifier.py +++ b/tests/test_intensifier/test_abstract_intensifier.py @@ -109,6 +109,33 @@ def test_incumbent_selection_multi_objective(make_scenario, configspace_small, m assert intensifier.get_incumbents() == [config] +def test_config_rejection_single_objective(configspace_small, make_scenario): + """Tests whether configs are rejected properly if they are worse than the incumbent.""" + scenario = make_scenario(configspace_small, use_instances=False) + runhistory = RunHistory() + intensifier = Intensifier(scenario=scenario) + intensifier.runhistory = runhistory + + configs = configspace_small.sample_configuration(3) + + runhistory.add(config=configs[0], cost=5, time=0.0, seed=0, status=StatusType.SUCCESS, force_update=True) + intensifier.update_incumbents(configs[0]) + + assert intensifier._rejected_config_ids == [] + + # add config that yielded better results, updating incumbent and sending prior incumbent to rejected + runhistory.add(config=configs[1], cost=1, time=0.0, seed=0, status=StatusType.SUCCESS, force_update=True) + intensifier.update_incumbents(config=configs[1]) + + assert intensifier._rejected_config_ids == [1] + + # add config that is no better should thus go to rejected + runhistory.add(config=configs[2], cost=1, time=0.0, seed=0, status=StatusType.SUCCESS, force_update=True) + intensifier.update_incumbents(config=configs[2]) + + assert intensifier._rejected_config_ids == [1, 3] + + def test_incumbent_differences(make_scenario, configspace_small): pass diff --git a/tests/test_intensifier/test_hyperband_utils.py b/tests/test_intensifier/test_hyperband_utils.py new file mode 100644 index 0000000000..33179d0ad1 --- /dev/null +++ b/tests/test_intensifier/test_hyperband_utils.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from smac.intensifier.hyperband_utils import ( + determine_HB, + determine_hyperband_for_multifidelity, + get_n_trials_for_hyperband_multifidelity, +) + + +def test_determine_HB(): + min_budget = 1.0 + max_budget = 81.0 + eta = 3 + + result = determine_HB(min_budget=min_budget, max_budget=max_budget, eta=eta) + + # Follow algorithm (not the table!) from https://arxiv.org/pdf/1603.06560.pdf (see https://github.com/automl/SMAC3/issues/977) + expected_max_iterations = {0: 5, 1: 4, 2: 3, 3: 2, 4: 1} + expected_n_configs_in_stage = { + 0: [81, 27, 9, 3, 1], + 1: [34, 11, 3, 1], + 2: [15, 5, 1], + 3: [8, 2], + 4: [5], + } + expected_budgets_in_stage = { + 0: [1, 3, 9, 27, 81], + 1: [3, 9, 27, 81], + 2: [9, 27, 81], + 3: [27, 81], + 4: [81], + } + expected_trials_used = 206 + expected_budget_used = 547 + expected_number_of_brackets = 5 + + assert result["max_iterations"] == expected_max_iterations + assert result["n_configs_in_stage"] == expected_n_configs_in_stage + assert result["budgets_in_stage"] == expected_budgets_in_stage + assert result["trials_used"] == expected_trials_used + assert result["budget_used"] == expected_budget_used + assert result["number_of_brackets"] == expected_number_of_brackets + + +def test_determine_hyperband_for_multifidelity(): + total_budget = 1000.0 + min_budget = 1.0 + max_budget = 81.0 + eta = 3 + + result = determine_hyperband_for_multifidelity( + total_budget=total_budget, min_budget=min_budget, max_budget=max_budget, eta=eta + ) + + expected_n_trials = 206 + 137 # 206 trials for one full round, and additional trials for the remaining budget + + assert result["n_trials"] == expected_n_trials + assert result["total_budget"] == total_budget + assert result["eta"] == eta + assert result["min_budget"] == min_budget + assert result["max_budget"] == max_budget + + +def test_get_n_trials_for_hyperband_multifidelity(): + total_budget = 1000.0 + min_budget = 1.0 + max_budget = 81.0 + eta = 3 + + n_trials = get_n_trials_for_hyperband_multifidelity( + total_budget=total_budget, min_budget=min_budget, max_budget=max_budget, eta=eta + ) + + assert n_trials == (206 + 137) diff --git a/tests/test_model/_test_gp_gpytorch.py b/tests/test_model/_test_gp_gpytorch.py index 95f3a59d89..d9e90f9b3a 100644 --- a/tests/test_model/_test_gp_gpytorch.py +++ b/tests/test_model/_test_gp_gpytorch.py @@ -325,7 +325,6 @@ def test_sampling_shape(self): X = np.arange(-5, 5, 0.1).reshape((-1, 1)) X_test = np.arange(-5.05, 5.05, 0.1).reshape((-1, 1)) for shape in (None, (-1, 1)): - if shape is None: y = np.sin(X).flatten() else: diff --git a/tests/test_model/test_gp.py b/tests/test_model/test_gp.py index 0c72eb2fa5..9e21925bc1 100644 --- a/tests/test_model/test_gp.py +++ b/tests/test_model/test_gp.py @@ -240,7 +240,6 @@ def __call__(self, X, eval_gradient=True, clone_kernel=True): raise np.linalg.LinAlgError with patch.object(sklearn.gaussian_process.GaussianProcessRegressor, "log_marginal_likelihood", Dummy().__call__): - seed = 1 rs = np.random.RandomState(seed) X, Y, n_dims = get_cont_data(rs) @@ -265,7 +264,6 @@ def __call__(self, X, Y=None): dummy = Dummy() with patch.object(GaussianProcess, "predict", dummy.__call__): - seed = 1 rs = np.random.RandomState(seed) @@ -375,7 +373,6 @@ def test_sampling_shape(): X = np.arange(-5, 5, 0.1).reshape((-1, 1)) X_test = np.arange(-5.05, 5.05, 0.1).reshape((-1, 1)) for shape in (None, (-1, 1)): - if shape is None: y = np.sin(X).flatten() else: diff --git a/tests/test_model/test_rf.py b/tests/test_model/test_rf.py index a30995b237..c81549a16e 100644 --- a/tests/test_model/test_rf.py +++ b/tests/test_model/test_rf.py @@ -127,7 +127,6 @@ def test_predict_marginalized(): def test_predict_marginalized_mocked(): - rs = np.random.RandomState(1) F = {} for i in range(10): @@ -257,24 +256,36 @@ def test_with_ordinal(): def test_impute_inactive_hyperparameters(): cs = ConfigurationSpace(seed=0) - a = cs.add_hyperparameter(CategoricalHyperparameter("a", [0, 1])) + a = cs.add_hyperparameter(CategoricalHyperparameter("a", [0, 1, 2])) b = cs.add_hyperparameter(CategoricalHyperparameter("b", [0, 1])) c = cs.add_hyperparameter(UniformFloatHyperparameter("c", 0, 1)) + d = cs.add_hyperparameter(OrdinalHyperparameter("d", [0, 1, 2])) cs.add_condition(EqualsCondition(b, a, 1)) cs.add_condition(EqualsCondition(c, a, 0)) + cs.add_condition(EqualsCondition(d, a, 2)) configs = cs.sample_configuration(size=100) config_array = convert_configurations_to_array(configs) for line in config_array: if line[0] == 0: assert np.isnan(line[1]) + assert np.isnan(line[3]) elif line[0] == 1: assert np.isnan(line[2]) + assert np.isnan(line[3]) + elif line[0] == 2: + assert np.isnan(line[1]) + assert np.isnan(line[2]) model = RandomForest(configspace=cs) config_array = model._impute_inactive(config_array) for line in config_array: if line[0] == 0: assert line[1] == 2 + assert line[3] == 3 elif line[0] == 1: assert line[2] == -1 + assert line[3] == 3 + elif line[0] == 2: + assert line[1] == 2 + assert line[2] == -1 diff --git a/tests/test_runhistory/test_runhistory.py b/tests/test_runhistory/test_runhistory.py index a1b7616ca4..428ec54adc 100644 --- a/tests/test_runhistory/test_runhistory.py +++ b/tests/test_runhistory/test_runhistory.py @@ -75,7 +75,6 @@ def test_add_and_pickle(runhistory, config1): def test_illegal_input(runhistory): - with pytest.raises(TypeError, match="Configuration must not be None."): runhistory.add(config=None, cost=1.23, time=2.34, status=StatusType.SUCCESS) @@ -87,7 +86,6 @@ def test_illegal_input(runhistory): def test_add_multiple_times(runhistory, config1): - for i in range(5): runhistory.add( config=config1, @@ -294,7 +292,6 @@ def test_full_update2(runhistory, config1, config2): def test_incremental_update(runhistory, config1): - runhistory.add( config=config1, cost=10, @@ -319,7 +316,6 @@ def test_incremental_update(runhistory, config1): def test_multiple_budgets(runhistory, config1): - runhistory.add( config=config1, cost=10, @@ -382,7 +378,6 @@ def test_get_configs_per_budget(runhistory, config1, config2, config3): def test_json_origin(configspace_small, config1): - for i, origin in enumerate(["test_origin", None]): config1.origin = origin runhistory = RunHistory() diff --git a/tests/test_runhistory/test_runhistory_encoder.py b/tests/test_runhistory/test_runhistory_encoder.py index 7b30bb76a5..ac9824534c 100644 --- a/tests/test_runhistory/test_runhistory_encoder.py +++ b/tests/test_runhistory/test_runhistory_encoder.py @@ -1,5 +1,7 @@ import numpy as np import pytest +from ConfigSpace import Configuration +from ConfigSpace.hyperparameters import CategoricalHyperparameter from smac.multi_objective.aggregation_strategy import MeanAggregationStrategy from smac.runhistory.encoder import ( @@ -41,62 +43,101 @@ def test_transform(runhistory, make_scenario, configspace_small, configs): # Normal encoder encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory + + # TODO: Please replace with the more general solution once ConfigSpace 1.0 + # upper = np.array([hp.upper_vectorized for hp in space.values()]) + # lower = np.array([hp.lower_vectorized for hp in space.values()]) + # - + # Categoricals are upperbounded by their size, rest of hyperparameters are + # upperbounded by 1. + upper_bounds = { + hp.name: (hp.size - 1) if isinstance(hp, CategoricalHyperparameter) else 1.0 + for hp in configspace_small.get_hyperparameters() + } + # Need to ensure they match the order in the Configuration vectorized form + sorted_by_indices = sorted( + upper_bounds.items(), + key=lambda x: configspace_small._hyperparameter_idx[x[0]], + ) + upper = np.array([upper_bound for _, upper_bound in sorted_by_indices]) + lower = 0.0 + X1, Y1 = encoder.transform() assert Y1.tolist() == [[1.0], [5.0]] - assert ((X1 <= 1.0) & (X1 >= 0.0)).all() + assert ((X1 <= upper) & (X1 >= lower)).all() # Log encoder encoder = RunHistoryLogEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() encoder = RunHistoryLogScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() encoder = RunHistoryScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() encoder = RunHistoryInverseScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() encoder = RunHistorySqrtScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() encoder = RunHistoryEIPSEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS]) encoder.runhistory = runhistory X, Y = encoder.transform() assert Y.tolist() != Y1.tolist() - assert ((X <= 1.0) & (X >= 0.0)).all() + assert ((X <= upper) & (X >= lower)).all() -def test_transform_conditionals(runhistory, make_scenario, configspace_large, configs): - configs = configspace_large.sample_configuration(20) +def test_transform_conditionals(runhistory, make_scenario, configspace_large): scenario = make_scenario(configspace_large) + config_1 = Configuration( + configspace_large, + values={ + "activation": "tanh", + "n_layer": 5, + "n_neurons": 27, + "solver": "lbfgs", + }, + ) + config_2 = Configuration( + configspace_large, + values={ + "activation": "tanh", + "batch_size": 47, + "learning_rate": "adaptive", + "learning_rate_init": 0.6673206111956781, + "n_layer": 3, + "n_neurons": 88, + "solver": "sgd", + }, + ) runhistory.add( - config=configs[0], + config=config_1, cost=1, time=1, status=StatusType.SUCCESS, ) runhistory.add( - config=configs[2], + config=config_2, cost=5, time=4, status=StatusType.SUCCESS, diff --git a/tests/test_utils/test_numpy_encoder.py b/tests/test_utils/test_numpy_encoder.py new file mode 100644 index 0000000000..9074e3cd80 --- /dev/null +++ b/tests/test_utils/test_numpy_encoder.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import json + +import numpy as np +import pytest + +from smac.utils.numpyencoder import NumpyEncoder + + +# Test cases for NumpyEncoder +def test_numpy_encoder(): + data = { + "int": np.int32(1), + "float": np.float32(1.23), + "complex": np.complex64(1 + 2j), + "array": np.array([1, 2, 3]), + "bool": np.bool_(True), + "void": np.void(b"void"), + } + + expected_output = { + "int": 1, + "float": 1.23, + "complex": {"real": 1.0, "imag": 2.0}, + "array": [1, 2, 3], + "bool": True, + "void": None, + } + + encoded_data = json.dumps(data, cls=NumpyEncoder) + decoded_data = json.loads(encoded_data) + + assert np.isclose(decoded_data["float"], expected_output["float"]) # float ist not exactly the same + del decoded_data["float"] + del expected_output["float"] + assert decoded_data == expected_output + + +# Test if default method raises TypeError for unsupported types +def test_numpy_encoder_unsupported_type(): + with pytest.raises(TypeError): + json.dumps(set([1, 2, 3]), cls=NumpyEncoder)