diff --git a/CHANGELOG.md b/CHANGELOG.md index 46b37206e..c66a8ef73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Dependencies - Update numpy NaN (#1122) and restrict numpy and ConfigSpace versions +- Add example to specify total budget (fidelity units) instead of n_trials for multi-fidelity/Hyperband (#1121) # 2.1.0 diff --git a/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py b/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py new file mode 100644 index 000000000..492cd7f46 --- /dev/null +++ b/examples/2_multi_fidelity/3_specify_HB_via_total_budget.py @@ -0,0 +1,112 @@ +""" +Specify Number of Trials via a Total Budget in Hyperband +^^^^^^^^^^^^^^^^^^ +This example uses a dummy function but illustrates how to setup Hyperband if you +want to specify a total optimization budget in terms of fidelity units. + +In Hyperband, normally SMAC calculates a typical Hyperband round. +If the number of trials is not used up by one single round, the next round is started. +Instead of specifying the number of trial beforehand, specify the total budget +in terms of the fidelity units and let SMAC calculate how many trials that would be. + + +""" +from __future__ import annotations + +import numpy as np +from ConfigSpace import Configuration, ConfigurationSpace, Float +from matplotlib import pyplot as plt + +from smac import MultiFidelityFacade, RunHistory, Scenario +from smac.intensifier.hyperband_utils import get_n_trials_for_hyperband_multifidelity + +__copyright__ = "Copyright 2021, AutoML.org Freiburg-Hannover" +__license__ = "3-clause BSD" + + +class QuadraticFunction: + max_budget = 500 + + @property + def configspace(self) -> ConfigurationSpace: + cs = ConfigurationSpace(seed=0) + x = Float("x", (-5, 5), default=-5) + cs.add_hyperparameters([x]) + + return cs + + def train(self, config: Configuration, seed: int = 0, budget: float | None = None) -> float: + """Returns the y value of a quadratic function with a minimum we know to be at x=0.""" + x = config["x"] + + if budget is None: + multiplier = 1 + else: + multiplier = 1 + budget / self.max_budget + + return x**2 * multiplier + + +def plot(runhistory: RunHistory, incumbent: Configuration) -> None: + plt.figure() + + # Plot ground truth + x = list(np.linspace(-5, 5, 100)) + y = [xi * xi for xi in x] + plt.plot(x, y) + + # Plot all trials + for k, v in runhistory.items(): + config = runhistory.get_config(k.config_id) + x = config["x"] + y = v.cost # type: ignore + plt.scatter(x, y, c="blue", alpha=0.1, zorder=9999, marker="o") + + # Plot incumbent + plt.scatter(incumbent["x"], incumbent["x"] * incumbent["x"], c="red", zorder=10000, marker="x") + + plt.show() + + +if __name__ == "__main__": + model = QuadraticFunction() + + min_budget = 10 # minimum budget per trial + max_budget = 500 # maximum budget per trial + eta = 3 # standard HB parameter influencing the number of stages + + # Let's calculate how many trials we need to exhaust the total optimization budget (in terms of + # fidelity units) + n_trials = get_n_trials_for_hyperband_multifidelity( + total_budget=10000, # this is the total optimization budget we specify in terms of fidelity units + min_budget=min_budget, # This influences the Hyperband rounds, minimum budget per trial + max_budget=max_budget, # This influences the Hyperband rounds, maximum budget per trial + eta=eta, # This influences the Hyperband rounds + print_summary=True, + ) + + # Scenario object specifying the optimization "environment" + scenario = Scenario( + model.configspace, deterministic=True, n_trials=n_trials, min_budget=min_budget, max_budget=max_budget + ) + + # Now we use SMAC to find the best hyperparameters + smac = MultiFidelityFacade( + scenario, + model.train, # We pass the target function here + overwrite=True, # Overrides any previous results that are found that are inconsistent with the meta-data + intensifier=MultiFidelityFacade.get_intensifier(scenario=scenario, eta=eta), + ) + + incumbent = smac.optimize() + + # Get cost of default configuration + default_cost = smac.validate(model.configspace.get_default_configuration()) + print(f"Default cost: {default_cost}") + + # Let's calculate the cost of the incumbent + incumbent_cost = smac.validate(incumbent) + print(f"Incumbent cost: {incumbent_cost}") + + # Let's plot it too + plot(smac.runhistory, incumbent) diff --git a/smac/intensifier/hyperband_utils.py b/smac/intensifier/hyperband_utils.py new file mode 100644 index 000000000..ad8773a71 --- /dev/null +++ b/smac/intensifier/hyperband_utils.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import numpy as np + +from smac.intensifier.successive_halving import SuccessiveHalving + + +def determine_HB(min_budget: float, max_budget: float, eta: int = 3) -> dict: + """Determine one Hyperband round + + Parameters + ---------- + min_budget : float + Minimum budget per trial in fidelity units + max_budget : float + Maximum budget per trial in fidelity units + eta : int, defaults to 3 + Input that controls the proportion of configurations discarded in each round of Successive Halving. + + Returns + ------- + dict + Info about the Hyperband round + "max_iterations" + "n_configs_in_stage" + "budgets_in_stage" + "trials_used" + "budget_used" + "number_of_brackets" + + """ + _s_max = SuccessiveHalving._get_max_iterations(eta, max_budget, min_budget) + + _max_iterations: dict[int, int] = {} + _n_configs_in_stage: dict[int, list] = {} + _budgets_in_stage: dict[int, list] = {} + + for i in range(_s_max + 1): + max_iter = _s_max - i + + _budgets_in_stage[i], _n_configs_in_stage[i] = SuccessiveHalving._compute_configs_and_budgets_for_stages( + eta, max_budget, max_iter, _s_max + ) + _max_iterations[i] = max_iter + 1 + + total_trials = np.sum([np.sum(v) for v in _n_configs_in_stage.values()]) + total_budget = np.sum([np.sum(v) for v in _budgets_in_stage.values()]) + + return { + "max_iterations": _max_iterations, + "n_configs_in_stage": _n_configs_in_stage, + "budgets_in_stage": _budgets_in_stage, + "trials_used": total_trials, + "budget_used": total_budget, + "number_of_brackets": _s_max, + } + + +def determine_hyperband_for_multifidelity( + total_budget: float, min_budget: float, max_budget: float, eta: int = 3 +) -> dict: + """Determine how many Hyperband rounds should happen based on a total budget + + Parameters + ---------- + total_budget : float + Total budget for the complete optimization in fidelity units + min_budget : float + Minimum budget per trial in fidelity units + max_budget : float + Maximum budget per trial in fidelity units + eta : int, defaults to 3 + Input that controls the proportion of configurations discarded in each round of Successive Halving. + + Returns + ------- + dict + Info about the Hyperband round + "max_iterations" + "n_configs_in_stage" + "budgets_in_stage" + "trials_used" + "budget_used" + "number_of_brackets" + + """ + # Determine the HB + hyperband_round = determine_HB(eta=eta, min_budget=min_budget, max_budget=max_budget) + + # Calculate how many HB rounds we can have + budget_used_per_hyperband_round = hyperband_round["budget_used"] + number_of_full_hb_rounds = int(np.floor(total_budget / budget_used_per_hyperband_round)) + remaining_budget = total_budget % budget_used_per_hyperband_round + trials_used_per_hb_round = hyperband_round["trials_used"] + n_configs_in_stage = hyperband_round["n_configs_in_stage"] + budgets_in_stage = hyperband_round["budgets_in_stage"] + + remaining_trials = 0 + for stage in n_configs_in_stage.keys(): + B = budgets_in_stage[stage] + C = n_configs_in_stage[stage] + for b, c in zip(B, C): + # How many trials are left? + # If b * c is lower than remaining budget, we can add full c + # otherwise we need to find out how many trials we can do with this budget + remaining_trials += min(c, int(np.floor(remaining_budget / b))) + # We cannot go lower than 0 + # If we are in the case of b*c > remaining_budget, we will not have any + # budget left. We can not add full c but the number of trials that still fit + remaining_budget = max(0, remaining_budget - b * c) + + # print(stage, b, c) + # print("-"*20, remaining_trials, remaining_budget) + + n_trials = int(number_of_full_hb_rounds * trials_used_per_hb_round + remaining_trials) + + hyperband_info = hyperband_round + hyperband_info["n_trials"] = n_trials + hyperband_info["total_budget"] = total_budget + hyperband_info["eta"] = eta + hyperband_info["min_budget"] = min_budget + hyperband_info["max_budget"] = max_budget + + return hyperband_info + + +def print_hyperband_summary(hyperband_info: dict) -> None: + """Print summary about Hyperband as used in the MultiFidelityFacade + + Parameters + ---------- + hyperband_info : dict + Info dict about Hyperband + """ + print("-" * 30, "HYPERBAND IN MULTI-FIDELITY", "-" * 30) + print("total budget:\t\t", hyperband_info["total_budget"]) + print("total number of trials:\t", hyperband_info["n_trials"]) + print("number of HB rounds:\t", hyperband_info["total_budget"] / hyperband_info["budget_used"]) + print() + + print("\t~~~~~~~~~~~~HYPERBAND ROUND") + print("\teta:\t\t\t\t\t", hyperband_info["eta"]) + print("\tmin budget per trial:\t\t\t", hyperband_info["min_budget"]) + print("\tmax budget per trial:\t\t\t", hyperband_info["max_budget"]) + print("\ttotal number of trials per HB round:\t", hyperband_info["trials_used"]) + print("\tbudget used per HB round:\t\t", hyperband_info["budget_used"]) + print("\tnumber of brackets:\t\t\t", hyperband_info["number_of_brackets"]) + print("\tbudgets per stage:\t\t\t", hyperband_info["budgets_in_stage"]) + print("\tn configs per stage:\t\t\t", hyperband_info["n_configs_in_stage"]) + print("-" * (2 * 30 + len("HYPERBAND IN MULTI-FIDELITY") + 2)) + + +def get_n_trials_for_hyperband_multifidelity( + total_budget: float, min_budget: float, max_budget: float, eta: int = 3, print_summary: bool = True +) -> int: + """Caculate the number of trials needed for multi-fidelity optimization + + Specify the total budget and find out how many trials that equals. + + Parameters + ---------- + total_budget : float + Total budget for the complete optimization in fidelity units + min_budget : float + Minimum budget per trial in fidelity units + max_budget : float + Maximum budget per trial in fidelity units + eta : int, defaults to 3 + Input that controls the proportion of configurations discarded in each round of Successive Halving. + + Returns + ------- + int + Number of trials needed for the specified total budgets + """ + hyperband_info = determine_hyperband_for_multifidelity( + total_budget=total_budget, eta=eta, min_budget=min_budget, max_budget=max_budget + ) + if print_summary: + print_hyperband_summary(hyperband_info=hyperband_info) + return hyperband_info["n_trials"]