Skip to content

Commit

Permalink
Merge pull request #2284 from Avaiga/feature/#521-refactor-code-for-d…
Browse files Browse the repository at this point in the history 
…atabricks-integration

Feature/#521 refactor code for databricks integration
  • Loading branch information
@toan-quach
toan-quach authored Dec 3, 2024
2 parents 5a6ff90 + 0e445b9 commit 23b6901
Show file tree
Hide file tree
Showing 2 changed files with 64 additions and 30 deletions.
66 changes: 42 additions & 24 deletions taipy/core/config/checkers/_data_node_config_checker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
# specific language governing permissions and limitations under the License.

from datetime import timedelta
from typing import Dict, List, cast
from typing import Any, Callable, Dict, List, Tuple, cast

from taipy.common.config._config import _Config
from taipy.common.config.checker._checkers._config_checker import _ConfigChecker
Expand All @@ -23,6 +23,27 @@


class _DataNodeConfigChecker(_ConfigChecker):
_PROPERTIES_TYPES: Dict[str, List[Tuple[Any, List[str]]]] = {
DataNodeConfig._STORAGE_TYPE_VALUE_GENERIC: [
(
Callable,
[
DataNodeConfig._OPTIONAL_READ_FUNCTION_GENERIC_PROPERTY,
DataNodeConfig._OPTIONAL_WRITE_FUNCTION_GENERIC_PROPERTY,
],
)
],
DataNodeConfig._STORAGE_TYPE_VALUE_SQL: [
(
Callable,
[
DataNodeConfig._REQUIRED_WRITE_QUERY_BUILDER_SQL_PROPERTY,
DataNodeConfig._OPTIONAL_APPEND_QUERY_BUILDER_SQL_PROPERTY,
],
),
],
}

def __init__(self, config: _Config, collector: IssueCollector):
super().__init__(config, collector)

Expand All @@ -46,7 +67,7 @@ def _check(self) -> IssueCollector:
self._check_scope(data_node_config_id, data_node_config)
self._check_validity_period(data_node_config_id, data_node_config)
self._check_required_properties(data_node_config_id, data_node_config)
self._check_callable(data_node_config_id, data_node_config)
self._check_class_type(data_node_config_id, data_node_config)
self._check_generic_read_write_fct_and_args(data_node_config_id, data_node_config)
self._check_exposed_type(data_node_config_id, data_node_config)
return self._collector
Expand Down Expand Up @@ -196,28 +217,25 @@ def _check_generic_read_write_fct_and_args(self, data_node_config_id: str, data_
f"DataNodeConfig `{data_node_config_id}` must be populated with a Callable function.",
)

def _check_callable(self, data_node_config_id: str, data_node_config: DataNodeConfig):
properties_to_check = {
DataNodeConfig._STORAGE_TYPE_VALUE_GENERIC: [
DataNodeConfig._OPTIONAL_READ_FUNCTION_GENERIC_PROPERTY,
DataNodeConfig._OPTIONAL_WRITE_FUNCTION_GENERIC_PROPERTY,
],
DataNodeConfig._STORAGE_TYPE_VALUE_SQL: [
DataNodeConfig._REQUIRED_WRITE_QUERY_BUILDER_SQL_PROPERTY,
DataNodeConfig._OPTIONAL_APPEND_QUERY_BUILDER_SQL_PROPERTY,
],
}

if data_node_config.storage_type in properties_to_check.keys():
for prop_key in properties_to_check[data_node_config.storage_type]:
prop_value = data_node_config.properties.get(prop_key) if data_node_config.properties else None
if prop_value and not callable(prop_value):
self._error(
prop_key,
prop_value,
f"`{prop_key}` of DataNodeConfig `{data_node_config_id}` must be"
f" populated with a Callable function.",
)
def _check_class_type(self, data_node_config_id: str, data_node_config: DataNodeConfig):
if data_node_config.storage_type in self._PROPERTIES_TYPES.keys():
for class_type, prop_keys in self._PROPERTIES_TYPES[data_node_config.storage_type]:
for prop_key in prop_keys:
prop_value = data_node_config.properties.get(prop_key) if data_node_config.properties else None
if prop_value and not isinstance(prop_value, class_type):
self._error(
prop_key,
prop_value,
f"`{prop_key}` of DataNodeConfig `{data_node_config_id}` must be"
f" populated with a {'Callable' if class_type == Callable else class_type.__name__}.",
)
if class_type == Callable and callable(prop_value) and prop_value.__name__ == "<lambda>":
self._error(
prop_key,
prop_value,
f"`{prop_key}` of DataNodeConfig `{data_node_config_id}` must be"
f" populated with a serializable Callable function but not a lambda.",
)

def _check_exposed_type(self, data_node_config_id: str, data_node_config: DataNodeConfig):
if not isinstance(data_node_config.exposed_type, str):
Expand Down
28 changes: 22 additions & 6 deletions tests/core/config/checkers/test_data_node_config_checker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -513,12 +513,12 @@ def test_check_callable_properties(self, caplog):
Config.check()
assert len(Config._collector.errors) == 2
expected_error_message_1 = (
"`write_query_builder` of DataNodeConfig `new` must be populated with a Callable function."
"`write_query_builder` of DataNodeConfig `new` must be populated with a Callable."
" Current value of property `write_query_builder` is 1."
)
assert expected_error_message_1 in caplog.text
expected_error_message_2 = (
"`append_query_builder` of DataNodeConfig `new` must be populated with a Callable function."
"`append_query_builder` of DataNodeConfig `new` must be populated with a Callable."
" Current value of property `append_query_builder` is 2."
)
assert expected_error_message_2 in caplog.text
Expand All @@ -530,7 +530,7 @@ def test_check_callable_properties(self, caplog):
Config.check()
assert len(Config._collector.errors) == 1
expected_error_messages = [
"`write_fct` of DataNodeConfig `new` must be populated with a Callable function. Current value"
"`write_fct` of DataNodeConfig `new` must be populated with a Callable. Current value"
" of property `write_fct` is 12.",
]
assert all(message in caplog.text for message in expected_error_messages)
Expand All @@ -542,7 +542,7 @@ def test_check_callable_properties(self, caplog):
Config.check()
assert len(Config._collector.errors) == 1
expected_error_messages = [
"`read_fct` of DataNodeConfig `new` must be populated with a Callable function. Current value"
"`read_fct` of DataNodeConfig `new` must be populated with a Callable. Current value"
" of property `read_fct` is 5.",
]
assert all(message in caplog.text for message in expected_error_messages)
Expand All @@ -554,9 +554,9 @@ def test_check_callable_properties(self, caplog):
Config.check()
assert len(Config._collector.errors) == 2
expected_error_messages = [
"`write_fct` of DataNodeConfig `new` must be populated with a Callable function. Current value"
"`write_fct` of DataNodeConfig `new` must be populated with a Callable. Current value"
" of property `write_fct` is 9.",
"`read_fct` of DataNodeConfig `new` must be populated with a Callable function. Current value"
"`read_fct` of DataNodeConfig `new` must be populated with a Callable. Current value"
" of property `read_fct` is 5.",
]
assert all(message in caplog.text for message in expected_error_messages)
Expand All @@ -581,6 +581,22 @@ def test_check_callable_properties(self, caplog):
Config.check()
assert len(Config._collector.errors) == 0

config._sections[DataNodeConfig.name]["new"].storage_type = "generic"
config._sections[DataNodeConfig.name]["new"].properties = {"write_fct": lambda x: x, "read_fct": lambda y: y}
with pytest.raises(SystemExit):
Config._collector = IssueCollector()
Config.check()
assert len(Config._collector.errors) == 2
expected_error_messages = [
"`write_fct` of DataNodeConfig `new` must be populated with a serializable Callable function but"
" not a lambda. Current value of property `write_fct` is <function TestDataNodeConfigChecker."
"test_check_callable_properties.<locals>.<lambda>",
"`read_fct` of DataNodeConfig `new` must be populated with a serializable Callable function but"
" not a lambda. Current value of property `read_fct` is <function TestDataNodeConfigChecker."
"test_check_callable_properties.<locals>.<lambda>",
]
assert all(message in caplog.text for message in expected_error_messages)

def test_check_read_write_fct_args(self, caplog):
config = Config._applied_config
Config._compile_configs()
Expand Down

0 comments on commit 23b6901

Please sign in to comment.