From 4452c73ace5ce3f5621774d935e7cf6ad1bf3c4e Mon Sep 17 00:00:00 2001 From: Vikas Pandey Date: Wed, 23 Aug 2023 12:57:21 +0530 Subject: [PATCH 1/4] fix for the tuning param in the config spec, fix the neuralforecast missed import & metric based model selection for automlx --- ads/opctl/operator/lowcode/forecast/const.py | 10 ++++++++++ ads/opctl/operator/lowcode/forecast/model/automlx.py | 8 ++++++-- .../operator/lowcode/forecast/model/neuralprophet.py | 3 ++- ads/opctl/operator/lowcode/forecast/model/prophet.py | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/ads/opctl/operator/lowcode/forecast/const.py b/ads/opctl/operator/lowcode/forecast/const.py index 43902c12b..9e5bb6aa7 100644 --- a/ads/opctl/operator/lowcode/forecast/const.py +++ b/ads/opctl/operator/lowcode/forecast/const.py @@ -23,3 +23,13 @@ class SupportedMetrics(str, metaclass=ExtendedEnumMeta): RMSE = "rmse" MSE = "mse" SMAPE = "smape" + + +automlx_metric_dict = { + "smape": "neg_sym_mean_abs_percent_error", + "mape": "neg_sym_mean_abs_percent_error", + "mase": "neg_mean_abs_scaled_error", + "mae": "neg_mean_absolute_error", + "mse": "neg_mean_squared_error", + "rmse": "neg_root_mean_squared_error", +} diff --git a/ads/opctl/operator/lowcode/forecast/model/automlx.py b/ads/opctl/operator/lowcode/forecast/model/automlx.py index 1173e9358..d15622ccc 100644 --- a/ads/opctl/operator/lowcode/forecast/model/automlx.py +++ b/ads/opctl/operator/lowcode/forecast/model/automlx.py @@ -9,6 +9,7 @@ import pandas as pd import numpy as np from ads.common.decorator.runtime_dependency import runtime_dependency +from ads.opctl.operator.lowcode.forecast.const import automlx_metric_dict from sktime.forecasting.model_selection import temporal_train_test_split from ads.opctl import logger @@ -29,7 +30,8 @@ class AutoMLXOperatorModel(ForecastOperatorBaseModel): ), ) def _build_model(self) -> pd.DataFrame: - import automl + from automl import init + init(engine="local", check_deprecation_warnings=False) full_data_dict = self.full_data_dict @@ -59,7 +61,9 @@ def _build_model(self) -> pd.DataFrame: "" if y_train.index.is_monotonic else "NOT", "monotonic.", ) - model = automl.Pipeline(task="forecasting", n_algos_tuned=n_algos_tuned) + model = automl.Pipeline(task="forecasting", + n_algos_tuned=n_algos_tuned, + score_metric=automlx_metric_dict[self.spec.metric]) model.fit(X=y_train.drop(target, axis=1), y=pd.DataFrame(y_train[target])) logger.info("Selected model: {}".format(model.selected_model_)) logger.info( diff --git a/ads/opctl/operator/lowcode/forecast/model/neuralprophet.py b/ads/opctl/operator/lowcode/forecast/model/neuralprophet.py index 789530b07..b3a259244 100644 --- a/ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +++ b/ads/opctl/operator/lowcode/forecast/model/neuralprophet.py @@ -9,6 +9,7 @@ import numpy as np import optuna import pandas as pd +from neuralprophet import NeuralProphet from torch import Tensor from torchmetrics.regression import ( MeanAbsoluteError, @@ -166,7 +167,7 @@ def objective(trial): ) study.optimize( objective, - n_trials=self.spec.tuning.n_trials if self.spec.tunning else 10, + n_trials=self.spec.tuning.n_trials if self.spec.tuning else 10, n_jobs=-1, ) diff --git a/ads/opctl/operator/lowcode/forecast/model/prophet.py b/ads/opctl/operator/lowcode/forecast/model/prophet.py index 30cf67511..edab7f08d 100644 --- a/ads/opctl/operator/lowcode/forecast/model/prophet.py +++ b/ads/opctl/operator/lowcode/forecast/model/prophet.py @@ -144,7 +144,7 @@ def objective(trial): ) study.optimize( objective, - n_trials=self.spec.tuning.n_trials if self.spec.tunning else 10, + n_trials=self.spec.tuning.n_trials if self.spec.tuning else 10, n_jobs=-1, ) From ad8075bdead53617b070e6b0b4ba24f94bad5f1f Mon Sep 17 00:00:00 2001 From: Vikas Pandey Date: Thu, 24 Aug 2023 09:58:06 +0530 Subject: [PATCH 2/4] use upppercase constants, use get to set default value for constant map --- ads/opctl/operator/lowcode/forecast/const.py | 2 +- ads/opctl/operator/lowcode/forecast/model/automlx.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ads/opctl/operator/lowcode/forecast/const.py b/ads/opctl/operator/lowcode/forecast/const.py index 9e5bb6aa7..3a4161966 100644 --- a/ads/opctl/operator/lowcode/forecast/const.py +++ b/ads/opctl/operator/lowcode/forecast/const.py @@ -25,7 +25,7 @@ class SupportedMetrics(str, metaclass=ExtendedEnumMeta): SMAPE = "smape" -automlx_metric_dict = { +AUTOMLX_METRIC_MAP = { "smape": "neg_sym_mean_abs_percent_error", "mape": "neg_sym_mean_abs_percent_error", "mase": "neg_mean_abs_scaled_error", diff --git a/ads/opctl/operator/lowcode/forecast/model/automlx.py b/ads/opctl/operator/lowcode/forecast/model/automlx.py index d15622ccc..7cfaad97a 100644 --- a/ads/opctl/operator/lowcode/forecast/model/automlx.py +++ b/ads/opctl/operator/lowcode/forecast/model/automlx.py @@ -9,7 +9,7 @@ import pandas as pd import numpy as np from ads.common.decorator.runtime_dependency import runtime_dependency -from ads.opctl.operator.lowcode.forecast.const import automlx_metric_dict +from ads.opctl.operator.lowcode.forecast.const import AUTOMLX_METRIC_MAP from sktime.forecasting.model_selection import temporal_train_test_split from ads.opctl import logger @@ -63,7 +63,7 @@ def _build_model(self) -> pd.DataFrame: ) model = automl.Pipeline(task="forecasting", n_algos_tuned=n_algos_tuned, - score_metric=automlx_metric_dict[self.spec.metric]) + score_metric=AUTOMLX_METRIC_MAP.get(self.spec.metric, "smape")) model.fit(X=y_train.drop(target, axis=1), y=pd.DataFrame(y_train[target])) logger.info("Selected model: {}".format(model.selected_model_)) logger.info( From e479a0c034885b7f1b68656960f6c928c2699e24 Mon Sep 17 00:00:00 2001 From: Vikas Pandey Date: Fri, 25 Aug 2023 10:14:05 +0530 Subject: [PATCH 3/4] update the default to correct value --- ads/opctl/operator/lowcode/forecast/model/automlx.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ads/opctl/operator/lowcode/forecast/model/automlx.py b/ads/opctl/operator/lowcode/forecast/model/automlx.py index 7cfaad97a..501a4a37f 100644 --- a/ads/opctl/operator/lowcode/forecast/model/automlx.py +++ b/ads/opctl/operator/lowcode/forecast/model/automlx.py @@ -63,7 +63,8 @@ def _build_model(self) -> pd.DataFrame: ) model = automl.Pipeline(task="forecasting", n_algos_tuned=n_algos_tuned, - score_metric=AUTOMLX_METRIC_MAP.get(self.spec.metric, "smape")) + score_metric=AUTOMLX_METRIC_MAP.get(self.spec.metric, + "neg_sym_mean_abs_percent_error")) model.fit(X=y_train.drop(target, axis=1), y=pd.DataFrame(y_train[target])) logger.info("Selected model: {}".format(model.selected_model_)) logger.info( From 9f829a392e47143bc8ac3e23ad0de9d9559c0635 Mon Sep 17 00:00:00 2001 From: Vikas Pandey Date: Mon, 11 Sep 2023 10:26:03 +0530 Subject: [PATCH 4/4] add the default_trials to constants: --- ads/opctl/operator/lowcode/forecast/const.py | 1 + ads/opctl/operator/lowcode/forecast/model/neuralprophet.py | 3 ++- ads/opctl/operator/lowcode/forecast/model/prophet.py | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ads/opctl/operator/lowcode/forecast/const.py b/ads/opctl/operator/lowcode/forecast/const.py index 8794c7b46..a7bce0e09 100644 --- a/ads/opctl/operator/lowcode/forecast/const.py +++ b/ads/opctl/operator/lowcode/forecast/const.py @@ -35,3 +35,4 @@ class SupportedMetrics(str, metaclass=ExtendedEnumMeta): } MAX_COLUMNS_AUTOMLX = 15 +DEFAULT_TRIALS = 10 \ No newline at end of file diff --git a/ads/opctl/operator/lowcode/forecast/model/neuralprophet.py b/ads/opctl/operator/lowcode/forecast/model/neuralprophet.py index b3a259244..ceb122ec7 100644 --- a/ads/opctl/operator/lowcode/forecast/model/neuralprophet.py +++ b/ads/opctl/operator/lowcode/forecast/model/neuralprophet.py @@ -25,6 +25,7 @@ ) from ads.opctl import logger +from ...forecast.const import DEFAULT_TRIALS from .. import utils from .base_model import ForecastOperatorBaseModel @@ -167,7 +168,7 @@ def objective(trial): ) study.optimize( objective, - n_trials=self.spec.tuning.n_trials if self.spec.tuning else 10, + n_trials=self.spec.tuning.n_trials if self.spec.tuning else DEFAULT_TRIALS, n_jobs=-1, ) diff --git a/ads/opctl/operator/lowcode/forecast/model/prophet.py b/ads/opctl/operator/lowcode/forecast/model/prophet.py index edab7f08d..0312e6201 100644 --- a/ads/opctl/operator/lowcode/forecast/model/prophet.py +++ b/ads/opctl/operator/lowcode/forecast/model/prophet.py @@ -15,6 +15,7 @@ from ads.opctl import logger +from ...forecast.const import DEFAULT_TRIALS from .. import utils from .base_model import ForecastOperatorBaseModel @@ -144,7 +145,7 @@ def objective(trial): ) study.optimize( objective, - n_trials=self.spec.tuning.n_trials if self.spec.tuning else 10, + n_trials=self.spec.tuning.n_trials if self.spec.tuning else DEFAULT_TRIALS, n_jobs=-1, )