Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow loose reinitialization from runs also #1006

Draft
wants to merge 2 commits into
base: develop
Choose a base branch
from
Draft

Allow loose reinitialization from runs also #1006

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ecb6906
Allow loose reinitialization from runs also
PGijsbers Nov 18, 2020
7d97fcd
Initial unit test design
Neeratyoy Dec 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions openml/runs/functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ def get_run_trace(run_id: int) -> OpenMLRunTrace:
return run_trace


def initialize_model_from_run(run_id: int) -> Any:
def initialize_model_from_run(run_id: int, strict_version: bool = True) -> Any:
"""
Initialized a model based on a run_id (i.e., using the exact
same parameter settings)
Expand All @@ -331,12 +331,15 @@ def initialize_model_from_run(run_id: int) -> Any:
run_id : int
The Openml run_id

strict_version: bool (default=True)
See `flow_to_model` strict_version.

Returns
-------
model
"""
run = get_run(run_id)
return initialize_model(run.setup_id)
return initialize_model(run.setup_id, strict_version)


def initialize_model_from_trace(
Expand Down
11 changes: 6 additions & 5 deletions openml/setups/functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,16 +228,17 @@ def __list_setups(api_call, output_format="object"):
return setups


def initialize_model(setup_id: int) -> Any:
"""
Initialized a model based on a setup_id (i.e., using the exact
same parameter settings)
def initialize_model(setup_id: int, strict_version: bool = True) -> Any:
""" Initialized a model based on a setup_id (i.e., using the exact same parameter settings)

Parameters
----------
setup_id : int
The Openml setup_id

strict_version: bool (default=True)
See `flow_to_model` strict_version.

Returns
-------
model
Expand All @@ -256,7 +257,7 @@ def initialize_model(setup_id: int) -> Any:
subflow = flow
subflow.parameters[hyperparameter.parameter_name] = hyperparameter.value

model = flow.extension.flow_to_model(flow)
model = flow.extension.flow_to_model(flow, strict_version=strict_version)
return model


Expand Down
42 changes: 42 additions & 0 deletions tests/test_runs/test_run_functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -940,6 +940,48 @@ def test_initialize_model_from_run(self):
self.assertEqual(flowS.components["Imputer"].parameters["strategy"], '"most_frequent"')
self.assertEqual(flowS.components["VarianceThreshold"].parameters["threshold"], "0.05")

@unittest.skipIf(
LooseVersion(sklearn.__version__) < "0.20",
reason="sklearn flows and pipelines changed drastically post 0.20 version",
)
def test_initialize_model_from_run_nonstrict(self):
# available runs on the test server, sorted based on the sklearn versions it uses
sklearn_runs = {
"sklearn==0.23.2": [
100,
111,
112,
120,
123,
130,
153,
164,
166,
167,
168,
803,
806,
807,
808,
814,
1373,
1374,
1375,
1376,
1377,
1378,
],
"sklearn==0.23.1": [386, 480, 512, 516, 520, 772, 1085, 1124, 1178, 1646],
"sklearn==0.20.2": [481],
"sklearn==0.22.2": [1108, 1612, 1642],
"sklearn==0.21.0": [1838],
}
current_version = "sklearn=={}".format(LooseVersion(sklearn.__version__).vstring)
sklearn_runs.pop(current_version)
version_choice = np.random.choice(list(sklearn_runs.keys()))
run_id_choice = np.random.choice(sklearn_runs[version_choice])
_ = openml.runs.initialize_model_from_run(run_id=run_id_choice, strict_version=False)

@unittest.skipIf(
LooseVersion(sklearn.__version__) < "0.20",
reason="SimpleImputer doesn't handle mixed type DataFrame as input",
Expand Down