Skip to content

Commit

Permalink
Merge branch 'release/4.0' into bug/#1226-fix-checker-raised-incorrec…
Browse files Browse the repository at this point in the history
…tly-for-polars
  • Loading branch information
toan-quach authored Dec 10, 2024
2 parents 1990026 + 44c3663 commit e772df3
Show file tree
Hide file tree
Showing 11 changed files with 66 additions and 38 deletions.
3 changes: 3 additions & 0 deletions taipy/core/_entity/_properties.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

from taipy.common.config.common._template_handler import _TemplateHandler as _tpl

from ..common._utils import _normalize_path
from ..notification import EventOperation, Notifier, _make_event


Expand All @@ -26,6 +27,8 @@ def __init__(self, entity_owner, **kwargs):
self._pending_deletions = set()

def __setitem__(self, key, value):
if key == "path":
value = _normalize_path(value)
super(_Properties, self).__setitem__(key, value)

if hasattr(self, "_entity_owner"):
Expand Down
5 changes: 5 additions & 0 deletions taipy/core/common/_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
# specific language governing permissions and limitations under the License.

import functools
import re
import time
from collections import namedtuple
from importlib import import_module
Expand Down Expand Up @@ -79,4 +80,8 @@ def _fcts_to_dict(objs):
return [d for obj in objs if (d := _fct_to_dict(obj)) is not None]


def _normalize_path(path: str) -> str:
return re.sub(r"[\\]+", "/", path)


_Subscriber = namedtuple("_Subscriber", "callback params")
5 changes: 4 additions & 1 deletion taipy/core/config/data_node_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from taipy.common.config.common.scope import Scope
from taipy.common.config.section import Section

from ..common._utils import _normalize_path
from ..common._warnings import _warn_deprecated
from ..common.mongo_default_document import MongoDefaultDocument

Expand Down Expand Up @@ -271,6 +272,8 @@ def __init__(
self._storage_type = storage_type
self._scope = scope
self._validity_period = validity_period
if "path" in properties:
properties["path"] = _normalize_path(properties["path"])
super().__init__(id, **properties)

# modin exposed type is deprecated since taipy 3.1.0
Expand Down Expand Up @@ -318,7 +321,7 @@ def scope(self, val) -> None:

@property
def validity_period(self) -> Optional[timedelta]:
""" The validity period of the data nodes instantiated from the data node config.
"""The validity period of the data nodes instantiated from the data node config.
It corresponds to the duration since the last edit date for which the data node
can be considered valid. Once the validity period has passed, the data node is
Expand Down
10 changes: 6 additions & 4 deletions taipy/core/data/_file_datanode_mixin.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from taipy.common.logger._taipy_logger import _TaipyLogger

from .._entity._reload import _self_reload
from ..common._utils import _normalize_path
from ..reason import InvalidUploadFile, NoFileToDownload, NotAFile, ReasonCollection, UploadFileCanNotBeRead
from .data_node import DataNode
from .data_node_id import Edit
Expand Down Expand Up @@ -60,13 +61,14 @@ def is_generated(self) -> bool:
@_self_reload(DataNode._MANAGER_NAME)
def path(self) -> str:
"""The path to the file data of the data node."""
return self._path
return _normalize_path(self._path)

@path.setter
def path(self, value) -> None:
self._path = value
self.properties[self._PATH_KEY] = value
self.properties[self._IS_GENERATED_KEY] = False
_path = _normalize_path(value)
self._path = _path
self.properties[self._PATH_KEY] = _path # type: ignore[attr-defined]
self.properties[self._IS_GENERATED_KEY] = False # type: ignore[attr-defined]

def is_downloadable(self) -> ReasonCollection:
"""Check if the data node is downloadable.
Expand Down
5 changes: 5 additions & 0 deletions tests/core/config/test_data_node_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -405,3 +405,8 @@ def test_clean_config():
assert dn1_config.validity_period is dn2_config.validity_period is None
assert dn1_config.default_path is dn2_config.default_path is None
assert dn1_config.properties == dn2_config.properties == {}


def test_normalize_path():
data_node_config = Config.configure_data_node(id="data_nodes1", storage_type="csv", path=r"data\file.csv")
assert data_node_config.path == "data/file.csv"
16 changes: 9 additions & 7 deletions tests/core/data/test_csv_data_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import dataclasses
import os
import pathlib
import re
import uuid
from datetime import datetime, timedelta
from time import sleep
Expand All @@ -25,6 +26,7 @@
from taipy.common.config import Config
from taipy.common.config.common.scope import Scope
from taipy.common.config.exceptions.exceptions import InvalidConfigurationId
from taipy.core.common._utils import _normalize_path
from taipy.core.data._data_manager import _DataManager
from taipy.core.data._data_manager_factory import _DataManagerFactory
from taipy.core.data.csv import CSVDataNode
Expand Down Expand Up @@ -129,7 +131,7 @@ def test_new_csv_data_node_with_existing_file_is_ready_for_reading(self):
)
def test_create_with_default_data(self, properties, exists):
dn = CSVDataNode("foo", Scope.SCENARIO, DataNodeId(f"dn_id_{uuid.uuid4()}"), properties=properties)
assert dn.path == os.path.join(Config.core.storage_folder.strip("/"), "csvs", dn.id + ".csv")
assert dn.path == f"{Config.core.storage_folder}csvs/{dn.id}.csv"
assert os.path.exists(dn.path) is exists

def test_set_path(self):
Expand Down Expand Up @@ -208,20 +210,20 @@ def test_is_not_downloadable_no_file(self):
reasons = dn.is_downloadable()
assert not reasons
assert len(reasons._reasons) == 1
assert str(NoFileToDownload(path, dn.id)) in reasons.reasons
assert str(NoFileToDownload(_normalize_path(path), dn.id)) in reasons.reasons

def test_is_not_downloadable_not_a_file(self):
path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample")
dn = CSVDataNode("foo", Scope.SCENARIO, properties={"path": path, "exposed_type": "pandas"})
reasons = dn.is_downloadable()
assert not reasons
assert len(reasons._reasons) == 1
assert str(NotAFile(path, dn.id)) in reasons.reasons
assert str(NotAFile(_normalize_path(path), dn.id)) in reasons.reasons

def test_get_downloadable_path(self):
path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.csv")
dn = CSVDataNode("foo", Scope.SCENARIO, properties={"path": path, "exposed_type": "pandas"})
assert dn._get_downloadable_path() == path
assert re.split(r"[\\/]", dn._get_downloadable_path()) == re.split(r"[\\/]", path)

def test_get_downloadable_path_with_not_existing_file(self):
dn = CSVDataNode("foo", Scope.SCENARIO, properties={"path": "NOT_EXISTING.csv", "exposed_type": "pandas"})
Expand All @@ -247,7 +249,7 @@ def test_upload(self, csv_file, tmpdir_factory):

assert_frame_equal(dn.read(), upload_content) # The content of the dn should change to the uploaded content
assert dn.last_edit_date > old_last_edit_date
assert dn.path == old_csv_path # The path of the dn should not change
assert dn.path == _normalize_path(old_csv_path) # The path of the dn should not change

def test_upload_with_upload_check_with_exception(self, csv_file, tmpdir_factory, caplog):
old_csv_path = tmpdir_factory.mktemp("data").join("df.csv").strpath
Expand Down Expand Up @@ -304,7 +306,7 @@ def check_data_column(upload_path, upload_data):

assert_frame_equal(dn.read(), old_data) # The content of the dn should not change when upload fails
assert dn.last_edit_date == old_last_edit_date # The last edit date should not change when upload fails
assert dn.path == old_csv_path # The path of the dn should not change
assert dn.path == _normalize_path(old_csv_path) # The path of the dn should not change

# The upload should succeed when check_data_column() return True
assert dn._upload(csv_file, upload_checker=check_data_column)
Expand Down Expand Up @@ -354,7 +356,7 @@ def check_data_is_positive(upload_path, upload_data):

np.array_equal(dn.read(), old_data) # The content of the dn should not change when upload fails
assert dn.last_edit_date == old_last_edit_date # The last edit date should not change when upload fails
assert dn.path == old_csv_path # The path of the dn should not change
assert dn.path == _normalize_path(old_csv_path) # The path of the dn should not change

# The upload should succeed when check_data_is_positive() return True
assert dn._upload(new_csv_path, upload_checker=check_data_is_positive)
2 changes: 1 addition & 1 deletion tests/core/data/test_data_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -667,7 +667,7 @@ def test_path_populated_with_config_default_path(self):
data_node.path = "baz.p"
assert data_node.path == "baz.p"

def test_track_edit(self):
def test_edit_edit_tracking(self):
dn_config = Config.configure_data_node("A")
data_node = _DataManager._bulk_get_or_create([dn_config])[dn_config]

Expand Down
16 changes: 9 additions & 7 deletions tests/core/data/test_excel_data_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

import os
import pathlib
import re
import uuid
from datetime import datetime, timedelta
from time import sleep
Expand All @@ -24,6 +25,7 @@

from taipy.common.config import Config
from taipy.common.config.common.scope import Scope
from taipy.core.common._utils import _normalize_path
from taipy.core.data._data_manager import _DataManager
from taipy.core.data._data_manager_factory import _DataManagerFactory
from taipy.core.data.data_node_id import DataNodeId
Expand Down Expand Up @@ -183,7 +185,7 @@ def test_set_path(self):
)
def test_create_with_default_data(self, properties, exists):
dn = ExcelDataNode("foo", Scope.SCENARIO, DataNodeId(f"dn_id_{uuid.uuid4()}"), properties=properties)
assert dn.path == os.path.join(Config.core.storage_folder.strip("/"), "excels", dn.id + ".xlsx")
assert dn.path == f"{Config.core.storage_folder}excels/{dn.id}.xlsx"
assert os.path.exists(dn.path) is exists

def test_read_write_after_modify_path(self):
Expand Down Expand Up @@ -423,20 +425,20 @@ def test_is_not_downloadable_no_file(self):
reasons = dn.is_downloadable()
assert not reasons
assert len(reasons._reasons) == 1
assert str(NoFileToDownload(path, dn.id)) in reasons.reasons
assert str(NoFileToDownload(_normalize_path(path), dn.id)) in reasons.reasons

def test_is_not_downloadable_not_a_file(self):
path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample")
dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": path, "exposed_type": "pandas"})
reasons = dn.is_downloadable()
assert not reasons
assert len(reasons._reasons) == 1
assert str(NotAFile(path, dn.id)) in reasons.reasons
assert str(NotAFile(_normalize_path(path), dn.id)) in reasons.reasons

def test_get_download_path(self):
path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/example.xlsx")
dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": path, "exposed_type": "pandas"})
assert dn._get_downloadable_path() == path
assert re.split(r"[\\/]", dn._get_downloadable_path()) == re.split(r"[\\/]", path)

def test_get_downloadable_path_with_not_existing_file(self):
dn = ExcelDataNode("foo", Scope.SCENARIO, properties={"path": "NOT_EXISTING.xlsx", "exposed_type": "pandas"})
Expand All @@ -457,7 +459,7 @@ def test_upload(self, excel_file, tmpdir_factory):

assert_frame_equal(dn.read()["Sheet1"], upload_content) # The data of dn should change to the uploaded content
assert dn.last_edit_date > old_last_edit_date
assert dn.path == old_xlsx_path # The path of the dn should not change
assert dn.path == _normalize_path(old_xlsx_path) # The path of the dn should not change

def test_upload_with_upload_check_pandas(self, excel_file, tmpdir_factory):
old_xlsx_path = tmpdir_factory.mktemp("data").join("df.xlsx").strpath
Expand Down Expand Up @@ -503,7 +505,7 @@ def check_data_column(upload_path, upload_data):

assert_frame_equal(dn.read()["Sheet1"], old_data) # The content of the dn should not change when upload fails
assert dn.last_edit_date == old_last_edit_date # The last edit date should not change when upload fails
assert dn.path == old_xlsx_path # The path of the dn should not change
assert dn.path == _normalize_path(old_xlsx_path) # The path of the dn should not change

# The upload should succeed when check_data_column() return True
assert dn._upload(excel_file, upload_checker=check_data_column)
Expand Down Expand Up @@ -552,7 +554,7 @@ def check_data_is_positive(upload_path, upload_data):

np.array_equal(dn.read()["Sheet1"], old_data) # The content of the dn should not change when upload fails
assert dn.last_edit_date == old_last_edit_date # The last edit date should not change when upload fails
assert dn.path == old_excel_path # The path of the dn should not change
assert dn.path == _normalize_path(old_excel_path) # The path of the dn should not change

# The upload should succeed when check_data_is_positive() return True
assert dn._upload(new_excel_path, upload_checker=check_data_is_positive)
14 changes: 8 additions & 6 deletions tests/core/data/test_json_data_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import json
import os
import pathlib
import re
import uuid
from dataclasses import dataclass
from enum import Enum
Expand All @@ -26,6 +27,7 @@
from taipy.common.config import Config
from taipy.common.config.common.scope import Scope
from taipy.common.config.exceptions.exceptions import InvalidConfigurationId
from taipy.core.common._utils import _normalize_path
from taipy.core.data._data_manager import _DataManager
from taipy.core.data._data_manager_factory import _DataManagerFactory
from taipy.core.data.data_node_id import DataNodeId
Expand Down Expand Up @@ -336,7 +338,7 @@ def test_filter(self, json_file):
)
def test_create_with_default_data(self, properties, exists):
dn = JSONDataNode("foo", Scope.SCENARIO, DataNodeId(f"dn_id_{uuid.uuid4()}"), properties=properties)
assert dn.path == os.path.join(Config.core.storage_folder.strip("/"), "jsons", dn.id + ".json")
assert dn.path == f"{Config.core.storage_folder}jsons/{dn.id}.json"
assert os.path.exists(dn.path) is exists

def test_set_path(self):
Expand Down Expand Up @@ -405,20 +407,20 @@ def test_is_not_downloadable_no_file(self):
reasons = dn.is_downloadable()
assert not reasons
assert len(reasons._reasons) == 1
assert str(NoFileToDownload(path, dn.id)) in reasons.reasons
assert str(NoFileToDownload(_normalize_path(path), dn.id)) in reasons.reasons

def is_not_downloadable_not_a_file(self):
path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/json")
dn = JSONDataNode("foo", Scope.SCENARIO, properties={"path": path})
reasons = dn.is_downloadable()
assert not reasons
assert len(reasons._reasons) == 1
assert str(NotAFile(path, dn.id)) in reasons.reasons
assert str(NotAFile(_normalize_path(path), dn.id)) in reasons.reasons

def test_get_download_path(self):
path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data_sample/json/example_dict.json")
dn = JSONDataNode("foo", Scope.SCENARIO, properties={"path": path})
assert dn._get_downloadable_path() == path
assert re.split(r"[\\/]", dn._get_downloadable_path()) == re.split(r"[\\/]", path)

def test_get_download_path_with_not_existed_file(self):
dn = JSONDataNode("foo", Scope.SCENARIO, properties={"path": "NOT_EXISTED.json"})
Expand All @@ -440,7 +442,7 @@ def test_upload(self, json_file, tmpdir_factory):

assert dn.read() == upload_content # The content of the dn should change to the uploaded content
assert dn.last_edit_date > old_last_edit_date
assert dn.path == old_json_path # The path of the dn should not change
assert dn.path == _normalize_path(old_json_path) # The path of the dn should not change

def test_upload_with_upload_check(self, json_file, tmpdir_factory):
old_json_path = tmpdir_factory.mktemp("data").join("df.json").strpath
Expand Down Expand Up @@ -486,7 +488,7 @@ def check_data_keys(upload_path, upload_data):

assert dn.read() == old_data # The content of the dn should not change when upload fails
assert dn.last_edit_date == old_last_edit_date # The last edit date should not change when upload fails
assert dn.path == old_json_path # The path of the dn should not change
assert dn.path == _normalize_path(old_json_path) # The path of the dn should not change

# The upload should succeed when check_data_keys() return True
assert dn._upload(json_file, upload_checker=check_data_keys)
Loading

0 comments on commit e772df3

Please sign in to comment.