From 65c607b8054bc852d9a86303f633f5b89b5d049c Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Thu, 18 Jan 2024 17:41:02 -0500 Subject: [PATCH 01/19] added parent path specification in save_pep function #32 --- docs/changelog.md | 4 ++++ pephubclient/files_manager.py | 20 +++++++++++++++----- pephubclient/pephubclient.py | 13 +++++++++++-- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index e2bb29a..d6c6474 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. +## [0.4.0] - 2024-XX-XX +### Added +- Added param parent dir where peps should be saved + ## [0.3.0] - 2024-01-17 ### Added - customization of the base pephub URL. [#22](https://github.com/pepkit/pephubclient/issues/22) diff --git a/pephubclient/files_manager.py b/pephubclient/files_manager.py index 3da0d19..a934186 100644 --- a/pephubclient/files_manager.py +++ b/pephubclient/files_manager.py @@ -29,17 +29,27 @@ def load_jwt_data_from_file(path: str) -> str: return f.read() @staticmethod - def create_project_folder(registry_path: RegistryPath) -> str: + def create_project_folder( + registry_path: RegistryPath, parent_path: str, just_name: bool = False + ) -> str: """ Create new project folder :param registry_path: project registry path + :param parent_path: parent path to create folder in + :param just_name: if True, create folder with just name, not full path :return: folder_path """ - folder_name = FilesManager._create_filename_to_save_downloaded_project( - registry_path - ) - folder_path = os.path.join(os.getcwd(), folder_name) + if just_name: + folder_name = registry_path.item + else: + folder_name = FilesManager._create_filename_to_save_downloaded_project( + registry_path + ) + if parent_path: + if not Path(parent_path).exists(): + raise OSError(f"Parent path does not exist. Provided path: {parent_path}") + folder_path = os.path.join(parent_path or os.getcwd(), folder_name) Path(folder_path).mkdir(parents=True, exist_ok=True) return folder_path diff --git a/pephubclient/pephubclient.py b/pephubclient/pephubclient.py index 118bdd2..0e8cb34 100644 --- a/pephubclient/pephubclient.py +++ b/pephubclient/pephubclient.py @@ -81,7 +81,9 @@ def pull(self, project_registry_path: str, force: Optional[bool] = False) -> Non ) self._save_raw_pep( - reg_path=project_registry_path, project_dict=project_dict, force=force + reg_path=project_registry_path, + project_dict=project_dict, + force=force, ) def load_project( @@ -252,16 +254,23 @@ def _save_raw_pep( reg_path: str, project_dict: dict, force: bool = False, + project_path: Optional[str] = None, + just_name: bool = False, ) -> None: """ Save project locally. + :param str reg_path: Project registry path in PEPhub (e.g. databio/base:default) :param dict project_dict: PEP dictionary (raw project) :param bool force: overwrite project if exists + :param str project_path: Path where project will be saved. By default, it will be saved in current directory. + :param bool just_name: If True, create project folder with just name, not full path :return: None """ reg_path_model = RegistryPath(**parse_registry_path(reg_path)) - folder_path = FilesManager.create_project_folder(registry_path=reg_path_model) + folder_path = FilesManager.create_project_folder( + registry_path=reg_path_model, parent_path=project_path, just_name=just_name + ) def full_path(fn: str) -> str: return os.path.join(folder_path, fn) From fae2fb523ff3df182fc820c3bcc63011949b7381 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 22 Jan 2024 13:09:14 -0500 Subject: [PATCH 02/19] updated pydantic model --- pephubclient/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pephubclient/constants.py b/pephubclient/constants.py index e36e66a..d625f58 100644 --- a/pephubclient/constants.py +++ b/pephubclient/constants.py @@ -14,10 +14,10 @@ class RegistryPath(BaseModel): - protocol: Optional[str] + protocol: Optional[str] = None namespace: str item: str - subitem: Optional[str] + subitem: Optional[str] = None tag: Optional[str] = "default" @field_validator("tag") From 12b578afa40c552b7248ab9296d890059ac35f9b Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 22 Jan 2024 13:28:43 -0500 Subject: [PATCH 03/19] fixed #33 --- pephubclient/helpers.py | 72 ++++++++++++++++++++++++ pephubclient/pephubclient.py | 106 +++++++++-------------------------- 2 files changed, 97 insertions(+), 81 deletions(-) diff --git a/pephubclient/helpers.py b/pephubclient/helpers.py index 41f06e9..07cc711 100644 --- a/pephubclient/helpers.py +++ b/pephubclient/helpers.py @@ -1,5 +1,14 @@ import json from typing import Any, Callable, Optional +import os +import pandas as pd +from peppy.const import ( + NAME_KEY, + DESC_KEY, + CONFIG_KEY, + SUBSAMPLE_RAW_LIST_KEY, + SAMPLE_RAW_DICT_KEY, +) import requests from requests.exceptions import ConnectionError @@ -9,6 +18,7 @@ from pephubclient.exceptions import PEPExistsError, ResponseError from pephubclient.constants import RegistryPath +from pephubclient.files_manager import FilesManager class RequestManager: @@ -103,3 +113,65 @@ def is_registry_path(input_string: str) -> bool: except (ValidationError, TypeError): return False return True + + +def save_raw_pep( + reg_path: str, + project_dict: dict, + force: bool = False, +) -> None: + """ + Save project locally. + + :param dict project_dict: PEP dictionary (raw project) + :param bool force: overwrite project if exists + :return: None + """ + reg_path_model = RegistryPath(**parse_registry_path(reg_path)) + folder_path = FilesManager.create_project_folder(registry_path=reg_path_model) + + def full_path(fn: str) -> str: + return os.path.join(folder_path, fn) + + project_name = project_dict[CONFIG_KEY][NAME_KEY] + sample_table_filename = "sample_table.csv" + yaml_full_path = full_path(f"{project_name}_config.yaml") + sample_full_path = full_path(sample_table_filename) + if not force: + extant = [p for p in [yaml_full_path, sample_full_path] if os.path.isfile(p)] + if extant: + raise PEPExistsError(f"{len(extant)} file(s) exist(s): {', '.join(extant)}") + + config_dict = project_dict.get(CONFIG_KEY) + config_dict[NAME_KEY] = project_name + config_dict[DESC_KEY] = project_dict[CONFIG_KEY][DESC_KEY] + config_dict["sample_table"] = sample_table_filename + + sample_pandas = pd.DataFrame(project_dict.get(SAMPLE_RAW_DICT_KEY, {})) + + subsample_list = [ + pd.DataFrame(sub_a) for sub_a in project_dict.get(SUBSAMPLE_RAW_LIST_KEY) or [] + ] + + filenames = [] + for idx, subsample in enumerate(subsample_list): + fn = f"subsample_table{idx + 1}.csv" + filenames.append(fn) + FilesManager.save_pandas(subsample, full_path(fn), not_force=False) + config_dict["subsample_table"] = filenames + + FilesManager.save_yaml(config_dict, yaml_full_path, not_force=False) + FilesManager.save_pandas(sample_pandas, sample_full_path, not_force=False) + + if config_dict.get("subsample_table"): + for number, subsample in enumerate(subsample_list): + FilesManager.save_pandas( + subsample, + os.path.join(folder_path, config_dict["subsample_table"][number]), + not_force=False, + ) + + MessageHandler.print_success( + f"Project was downloaded successfully -> {folder_path}" + ) + return None diff --git a/pephubclient/pephubclient.py b/pephubclient/pephubclient.py index 118bdd2..a12e21b 100644 --- a/pephubclient/pephubclient.py +++ b/pephubclient/pephubclient.py @@ -1,16 +1,10 @@ import json import os from typing import NoReturn, Optional, Literal +from typing_extensions import deprecated -import pandas as pd import peppy -from peppy.const import ( - NAME_KEY, - DESC_KEY, - CONFIG_KEY, - SUBSAMPLE_RAW_LIST_KEY, - SAMPLE_RAW_DICT_KEY, -) +from peppy.const import NAME_KEY import requests import urllib3 from pydantic import ValidationError @@ -25,11 +19,10 @@ ) from pephubclient.exceptions import ( IncorrectQueryStringError, - PEPExistsError, ResponseError, ) from pephubclient.files_manager import FilesManager -from pephubclient.helpers import MessageHandler, RequestManager +from pephubclient.helpers import MessageHandler, RequestManager, save_raw_pep from pephubclient.models import ( ProjectDict, ProjectUploadData, @@ -76,11 +69,11 @@ def pull(self, project_registry_path: str, force: Optional[bool] = False) -> Non :return: None """ jwt_data = FilesManager.load_jwt_data_from_file(self.PATH_TO_FILE_WITH_JWT) - project_dict = self._load_raw_pep( + project_dict = self.load_raw_pep( registry_path=project_registry_path, jwt_data=jwt_data ) - self._save_raw_pep( + save_raw_pep( reg_path=project_registry_path, project_dict=project_dict, force=force ) @@ -97,7 +90,7 @@ def load_project( :return Project: peppy project. """ jwt = FilesManager.load_jwt_data_from_file(self.PATH_TO_FILE_WITH_JWT) - raw_pep = self._load_raw_pep(project_registry_path, jwt, query_param) + raw_pep = self.load_raw_pep(project_registry_path, jwt, query_param) peppy_project = peppy.Project().from_dict(raw_pep) return peppy_project @@ -247,80 +240,32 @@ def find_project( project_list.append(ProjectAnnotationModel(**project_found)) return SearchReturnModel(**json.loads(decoded_response)) - @staticmethod - def _save_raw_pep( - reg_path: str, - project_dict: dict, - force: bool = False, - ) -> None: + @deprecated("This method is deprecated. Use load_raw_pep instead.") + def _load_raw_pep( + self, + registry_path: str, + jwt_data: Optional[str] = None, + query_param: Optional[dict] = None, + ) -> dict: """ - Save project locally. + !!! This method is deprecated. Use load_raw_pep instead. !!! - :param dict project_dict: PEP dictionary (raw project) - :param bool force: overwrite project if exists - :return: None + Request PEPhub and return the requested project as peppy.Project object. + + :param registry_path: Project namespace, eg. "geo/GSE124224:tag" + :param query_param: Optional variables to be passed to PEPhub + :param jwt_data: JWT token. + :return: Raw project in dict. """ - reg_path_model = RegistryPath(**parse_registry_path(reg_path)) - folder_path = FilesManager.create_project_folder(registry_path=reg_path_model) - - def full_path(fn: str) -> str: - return os.path.join(folder_path, fn) - - project_name = project_dict[CONFIG_KEY][NAME_KEY] - sample_table_filename = "sample_table.csv" - yaml_full_path = full_path(f"{project_name}_config.yaml") - sample_full_path = full_path(sample_table_filename) - if not force: - extant = [ - p for p in [yaml_full_path, sample_full_path] if os.path.isfile(p) - ] - if extant: - raise PEPExistsError( - f"{len(extant)} file(s) exist(s): {', '.join(extant)}" - ) - - config_dict = project_dict.get(CONFIG_KEY) - config_dict[NAME_KEY] = project_name - config_dict[DESC_KEY] = project_dict[CONFIG_KEY][DESC_KEY] - config_dict["sample_table"] = sample_table_filename - - sample_pandas = pd.DataFrame(project_dict.get(SAMPLE_RAW_DICT_KEY, {})) - - subsample_list = [ - pd.DataFrame(sub_a) - for sub_a in project_dict.get(SUBSAMPLE_RAW_LIST_KEY) or [] - ] - - filenames = [] - for idx, subsample in enumerate(subsample_list): - fn = f"subsample_table{idx + 1}.csv" - filenames.append(fn) - FilesManager.save_pandas(subsample, full_path(fn), not_force=False) - config_dict["subsample_table"] = filenames - - FilesManager.save_yaml(config_dict, yaml_full_path, not_force=False) - FilesManager.save_pandas(sample_pandas, sample_full_path, not_force=False) - - if config_dict.get("subsample_table"): - for number, subsample in enumerate(subsample_list): - FilesManager.save_pandas( - subsample, - os.path.join(folder_path, config_dict["subsample_table"][number]), - not_force=False, - ) - - MessageHandler.print_success( - f"Project was downloaded successfully -> {folder_path}" - ) - return None + return self.load_raw_pep(registry_path, jwt_data, query_param) - def _load_raw_pep( + def load_raw_pep( self, registry_path: str, jwt_data: Optional[str] = None, query_param: Optional[dict] = None, ) -> dict: - """project_name + """ Request PEPhub and return the requested project as peppy.Project object. :param registry_path: Project namespace, eg. "geo/GSE124224:tag" @@ -396,9 +341,8 @@ def _build_pull_request_url(self, query_param: dict = None) -> str: return PEPHUB_PEP_API_BASE_URL + endpoint - def _build_project_search_url( - self, namespace: str, query_param: dict = None - ) -> str: + @staticmethod + def _build_project_search_url(namespace: str, query_param: dict = None) -> str: """ Build request for searching projects form pephub From de713165504579db2fd45d34cc284500daabd318 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 23 Jan 2024 15:23:26 -0500 Subject: [PATCH 04/19] fixed #32 fixed #34 --- pephubclient/__init__.py | 10 ++- pephubclient/cli.py | 4 ++ pephubclient/files_manager.py | 47 ++++++------- pephubclient/helpers.py | 121 ++++++++++++++++++++++++++++++---- pephubclient/pephubclient.py | 96 +++++---------------------- tests/test_pephubclient.py | 35 ++++++++++ 6 files changed, 195 insertions(+), 118 deletions(-) diff --git a/pephubclient/__init__.py b/pephubclient/__init__.py index 8e2e1a5..c94b9ef 100644 --- a/pephubclient/__init__.py +++ b/pephubclient/__init__.py @@ -1,8 +1,16 @@ from pephubclient.pephubclient import PEPHubClient +from pephubclient.helpers import is_registry_path, save_pep __app_name__ = "pephubclient" __version__ = "0.2.2" __author__ = "Oleksandr Khoroshevskyi, Rafal Stepien" -__all__ = ["PEPHubClient", __app_name__, __author__, __version__] +__all__ = [ + "PEPHubClient", + __app_name__, + __author__, + __version__, + "is_registry_path", + "save_pep", +] diff --git a/pephubclient/cli.py b/pephubclient/cli.py index 7553062..7be8cfa 100644 --- a/pephubclient/cli.py +++ b/pephubclient/cli.py @@ -29,6 +29,8 @@ def logout(): def pull( project_registry_path: str, force: bool = typer.Option(False, help="Overwrite project if it exists."), + zip: bool = typer.Option(False, help="Save project as zip file."), + output: str = typer.Option(None, help="Output directory."), ): """ Download and save project locally. @@ -37,6 +39,8 @@ def pull( _client.pull, project_registry_path=project_registry_path, force=force, + output=output, + zip=zip, ) diff --git a/pephubclient/files_manager.py b/pephubclient/files_manager.py index a934186..84bebc5 100644 --- a/pephubclient/files_manager.py +++ b/pephubclient/files_manager.py @@ -4,6 +4,7 @@ import pandas import yaml +import zipfile from pephubclient.constants import RegistryPath from pephubclient.exceptions import PEPExistsError @@ -30,25 +31,21 @@ def load_jwt_data_from_file(path: str) -> str: @staticmethod def create_project_folder( - registry_path: RegistryPath, parent_path: str, just_name: bool = False + parent_path: str, + folder_name: str, ) -> str: """ Create new project folder - :param registry_path: project registry path :param parent_path: parent path to create folder in - :param just_name: if True, create folder with just name, not full path + :param folder_name: folder name :return: folder_path """ - if just_name: - folder_name = registry_path.item - else: - folder_name = FilesManager._create_filename_to_save_downloaded_project( - registry_path - ) if parent_path: if not Path(parent_path).exists(): - raise OSError(f"Parent path does not exist. Provided path: {parent_path}") + raise OSError( + f"Parent path does not exist. Provided path: {parent_path}" + ) folder_path = os.path.join(parent_path or os.getcwd(), folder_name) Path(folder_path).mkdir(parents=True, exist_ok=True) return folder_path @@ -73,20 +70,24 @@ def delete_file_if_exists(filename: str) -> None: with suppress(FileNotFoundError): os.remove(filename) - @staticmethod - def _create_filename_to_save_downloaded_project(registry_path: RegistryPath) -> str: - """ - Takes query string and creates output filename to save the project to. - - :param registry_path: Query string that was used to find the project. - :return: Filename uniquely identifying the project. - """ - filename = "_".join(filter(bool, [registry_path.namespace, registry_path.item])) - if registry_path.tag: - filename += f":{registry_path.tag}" - return filename - @staticmethod def check_writable(path: str, force: bool = True): if not force and os.path.isfile(path): raise PEPExistsError(f"File already exists and won't be updated: {path}") + + @staticmethod + def save_zip_file(files_dict: dict, file_path: str, force: bool = False) -> None: + """ + Save zip file with provided files as dict. + + :param files_dict: dict with files to save. e.g. {"file1.txt": "file1 content"} + :param file_path: filename to save zip file to + :param force: overwrite file if exists + :return: None + """ + FilesManager.check_writable(path=file_path, force=force) + with zipfile.ZipFile( + file_path, mode="w", compression=zipfile.ZIP_DEFLATED + ) as zf: + for name, res in files_dict.items(): + zf.writestr(name, str.encode(res)) diff --git a/pephubclient/helpers.py b/pephubclient/helpers.py index 07cc711..027c38f 100644 --- a/pephubclient/helpers.py +++ b/pephubclient/helpers.py @@ -1,5 +1,7 @@ import json -from typing import Any, Callable, Optional +from typing import Any, Callable, Optional, Union +import peppy +import yaml import os import pandas as pd from peppy.const import ( @@ -8,6 +10,8 @@ CONFIG_KEY, SUBSAMPLE_RAW_LIST_KEY, SAMPLE_RAW_DICT_KEY, + CFG_SAMPLE_TABLE_KEY, + CFG_SUBSAMPLE_TABLE_KEY, ) import requests @@ -19,6 +23,7 @@ from pephubclient.exceptions import PEPExistsError, ResponseError from pephubclient.constants import RegistryPath from pephubclient.files_manager import FilesManager +from pephubclient.models import ProjectDict class RequestManager: @@ -115,20 +120,69 @@ def is_registry_path(input_string: str) -> bool: return True -def save_raw_pep( - reg_path: str, - project_dict: dict, - force: bool = False, +def _build_filename(registry_path: RegistryPath) -> str: + """ + Takes query string and creates output filename to save the project to. + + :param registry_path: Query string that was used to find the project. + :return: Filename uniquely identifying the project. + """ + filename = "_".join(filter(bool, [registry_path.namespace, registry_path.item])) + if registry_path.tag: + filename += f"_{registry_path.tag}" + return filename + + +def _save_zip_pep(project: dict, zip_filepath: str, force: bool = False) -> None: + """ + Zip and save a project + + :param project: peppy project to zip + :param zip_filepath: path to save zip file + :param force: overwrite project if exists + """ + + content_to_zip = {} + config = project[CONFIG_KEY] + project_name = config[NAME_KEY] + + if project[SAMPLE_RAW_DICT_KEY] is not None: + config[CFG_SAMPLE_TABLE_KEY] = ["sample_table.csv"] + content_to_zip["sample_table.csv"] = pd.DataFrame( + project[SAMPLE_RAW_DICT_KEY] + ).to_csv(index=False) + + if project[SUBSAMPLE_RAW_LIST_KEY] is not None: + if not isinstance(project[SUBSAMPLE_RAW_LIST_KEY], list): + config[CFG_SUBSAMPLE_TABLE_KEY] = ["subsample_table1.csv"] + content_to_zip["subsample_table1.csv"] = pd.DataFrame( + project[SUBSAMPLE_RAW_LIST_KEY] + ).to_csv(index=False) + else: + config[CFG_SUBSAMPLE_TABLE_KEY] = [] + for number, file in enumerate(project[SUBSAMPLE_RAW_LIST_KEY]): + file_name = f"subsample_table{number + 1}.csv" + config[CFG_SUBSAMPLE_TABLE_KEY].append(file_name) + content_to_zip[file_name] = pd.DataFrame(file).to_csv(index=False) + + content_to_zip[f"{project_name}_config.yaml"] = yaml.dump(config, indent=4) + FilesManager.save_zip_file(content_to_zip, file_path=zip_filepath, force=force) + + MessageHandler.print_success(f"Project was saved successfully -> {zip_filepath}") + return None + + +def _save_unzipped_pep( + project_dict: dict, folder_path: str, force: bool = False ) -> None: """ - Save project locally. + Save unzipped project to specified folder - :param dict project_dict: PEP dictionary (raw project) - :param bool force: overwrite project if exists + :param project_dict: raw pep project + :param folder_path: path to save project + :param force: overwrite project if exists :return: None """ - reg_path_model = RegistryPath(**parse_registry_path(reg_path)) - folder_path = FilesManager.create_project_folder(registry_path=reg_path_model) def full_path(fn: str) -> str: return os.path.join(folder_path, fn) @@ -171,7 +225,48 @@ def full_path(fn: str) -> str: not_force=False, ) - MessageHandler.print_success( - f"Project was downloaded successfully -> {folder_path}" - ) + MessageHandler.print_success(f"Project was saved successfully -> {folder_path}") return None + + +def save_pep( + project: Union[dict, peppy.Project], + reg_path: str = None, + force: bool = False, + project_path: Optional[str] = None, + zip: bool = False, +) -> None: + """ + Save project locally. + + :param dict project: PEP dictionary (raw project) + :param str reg_path: Project registry path in PEPhub (e.g. databio/base:default). If not provided, + folder will be created with just project name. + :param bool force: overwrite project if exists + :param str project_path: Path where project will be saved. By default, it will be saved in current directory. + :param bool zip: If True, save project as zip file + :return: None + """ + if isinstance(project, peppy.Project): + project = project.to_dict(extended=True, orient="records") + + project = ProjectDict(**project).model_dump(by_alias=True) + + if not project_path: + project_path = os.getcwd() + + if reg_path: + file_name = _build_filename(RegistryPath(**parse_registry_path(reg_path))) + else: + file_name = project[CONFIG_KEY][NAME_KEY] + + if zip: + _save_zip_pep( + project, zip_filepath=f"{os.path.join(project_path, file_name)}.zip" + ) + return None + + folder_path = FilesManager.create_project_folder( + parent_path=project_path, folder_name=file_name + ) + _save_unzipped_pep(project, folder_path, force=force) diff --git a/pephubclient/pephubclient.py b/pephubclient/pephubclient.py index d9c021b..500d466 100644 --- a/pephubclient/pephubclient.py +++ b/pephubclient/pephubclient.py @@ -22,7 +22,7 @@ ResponseError, ) from pephubclient.files_manager import FilesManager -from pephubclient.helpers import MessageHandler, RequestManager, save_raw_pep +from pephubclient.helpers import MessageHandler, RequestManager, save_pep from pephubclient.models import ( ProjectDict, ProjectUploadData, @@ -60,12 +60,20 @@ def logout(self) -> NoReturn: """ FilesManager.delete_file_if_exists(self.PATH_TO_FILE_WITH_JWT) - def pull(self, project_registry_path: str, force: Optional[bool] = False) -> None: + def pull( + self, + project_registry_path: str, + force: Optional[bool] = False, + zip: Optional[bool] = False, + output: Optional[str] = None, + ) -> None: """ Download project locally :param str project_registry_path: Project registry path in PEPhub (e.g. databio/base:default) :param bool force: if project exists, overwrite it. + :param bool zip: if True, save project as zip file + :param str output: path where project will be saved :return: None """ jwt_data = FilesManager.load_jwt_data_from_file(self.PATH_TO_FILE_WITH_JWT) @@ -73,12 +81,12 @@ def pull(self, project_registry_path: str, force: Optional[bool] = False) -> Non registry_path=project_registry_path, jwt_data=jwt_data ) - self._save_raw_pep( + save_pep( + project=project_dict, reg_path=project_registry_path, - project_dict=project_dict, - force=force - save_raw_pep( - reg_path=project_registry_path, project_dict=project_dict, force=force + force=force, + project_path=output, + zip=zip, ) def load_project( @@ -244,80 +252,6 @@ def find_project( project_list.append(ProjectAnnotationModel(**project_found)) return SearchReturnModel(**json.loads(decoded_response)) - @staticmethod - def _save_raw_pep( - reg_path: str, - project_dict: dict, - force: bool = False, - project_path: Optional[str] = None, - just_name: bool = False, - ) -> None: - """ - Save project locally. - - :param str reg_path: Project registry path in PEPhub (e.g. databio/base:default) - :param dict project_dict: PEP dictionary (raw project) - :param bool force: overwrite project if exists - :param str project_path: Path where project will be saved. By default, it will be saved in current directory. - :param bool just_name: If True, create project folder with just name, not full path - :return: None - """ - reg_path_model = RegistryPath(**parse_registry_path(reg_path)) - folder_path = FilesManager.create_project_folder( - registry_path=reg_path_model, parent_path=project_path, just_name=just_name - ) - - def full_path(fn: str) -> str: - return os.path.join(folder_path, fn) - - project_name = project_dict[CONFIG_KEY][NAME_KEY] - sample_table_filename = "sample_table.csv" - yaml_full_path = full_path(f"{project_name}_config.yaml") - sample_full_path = full_path(sample_table_filename) - if not force: - extant = [ - p for p in [yaml_full_path, sample_full_path] if os.path.isfile(p) - ] - if extant: - raise PEPExistsError( - f"{len(extant)} file(s) exist(s): {', '.join(extant)}" - ) - - config_dict = project_dict.get(CONFIG_KEY) - config_dict[NAME_KEY] = project_name - config_dict[DESC_KEY] = project_dict[CONFIG_KEY][DESC_KEY] - config_dict["sample_table"] = sample_table_filename - - sample_pandas = pd.DataFrame(project_dict.get(SAMPLE_RAW_DICT_KEY, {})) - - subsample_list = [ - pd.DataFrame(sub_a) - for sub_a in project_dict.get(SUBSAMPLE_RAW_LIST_KEY) or [] - ] - - filenames = [] - for idx, subsample in enumerate(subsample_list): - fn = f"subsample_table{idx + 1}.csv" - filenames.append(fn) - FilesManager.save_pandas(subsample, full_path(fn), not_force=False) - config_dict["subsample_table"] = filenames - - FilesManager.save_yaml(config_dict, yaml_full_path, not_force=False) - FilesManager.save_pandas(sample_pandas, sample_full_path, not_force=False) - - if config_dict.get("subsample_table"): - for number, subsample in enumerate(subsample_list): - FilesManager.save_pandas( - subsample, - os.path.join(folder_path, config_dict["subsample_table"][number]), - not_force=False, - ) - - MessageHandler.print_success( - f"Project was downloaded successfully -> {folder_path}" - ) - return None - @deprecated("This method is deprecated. Use load_raw_pep instead.") def _load_raw_pep( self, diff --git a/tests/test_pephubclient.py b/tests/test_pephubclient.py index c35c8ac..b179b94 100644 --- a/tests/test_pephubclient.py +++ b/tests/test_pephubclient.py @@ -201,3 +201,38 @@ class TestHelpers: ) def test_is_registry_path(self, input_str, expected_output): assert is_registry_path(input_str) is expected_output + + @pytest.mark.skipif(True, reason="not implemented yet") + def test_save_zip_pep(self): + ... + + @pytest.mark.skipif(True, reason="not implemented yet") + def test_save_unzip_pep(self): + ... + + +@pytest.mark.skipif(True, reason="not implemented yet") +class TestSamplesModification: + def test_get_sumple(self): + ... + + def test_add_sample(self): + ... + + def test_remove_sample(self): + ... + + def test_update_sample(self): + ... + + +@pytest.mark.skipif(True, reason="not implemented yet") +class TestProjectVeiw: + def test_get_view(self): + ... + + def test_create_view(self): + ... + + def test_delete_view(self): + ... From e93238628c350299b254f4760f509cb77790dd95 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 23 Jan 2024 15:24:16 -0500 Subject: [PATCH 05/19] updated version --- pephubclient/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pephubclient/__init__.py b/pephubclient/__init__.py index c94b9ef..39deb45 100644 --- a/pephubclient/__init__.py +++ b/pephubclient/__init__.py @@ -2,7 +2,7 @@ from pephubclient.helpers import is_registry_path, save_pep __app_name__ = "pephubclient" -__version__ = "0.2.2" +__version__ = "0.4.0" __author__ = "Oleksandr Khoroshevskyi, Rafal Stepien" From bdbb7ab4a352c1d792fd3d93275947fa92c5f199 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 23 Jan 2024 16:10:24 -0500 Subject: [PATCH 06/19] Fixed saving force bug --- pephubclient/helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pephubclient/helpers.py b/pephubclient/helpers.py index 027c38f..7d582a1 100644 --- a/pephubclient/helpers.py +++ b/pephubclient/helpers.py @@ -262,7 +262,9 @@ def save_pep( if zip: _save_zip_pep( - project, zip_filepath=f"{os.path.join(project_path, file_name)}.zip" + project, + zip_filepath=f"{os.path.join(project_path, file_name)}.zip", + force=force, ) return None From b0888a454386ac996cb3332ca220f008ecc53299 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 23 Jan 2024 16:35:03 -0500 Subject: [PATCH 07/19] Added the skeleton for Views and Samples functions --- pephubclient/files_manager.py | 4 +++- pephubclient/pephubclient.py | 12 ++++++++++ pephubclient/samples/__init__.py | 3 +++ pephubclient/samples/samples.py | 38 ++++++++++++++++++++++++++++++++ pephubclient/views/__init__.py | 3 +++ pephubclient/views/views.py | 34 ++++++++++++++++++++++++++++ 6 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 pephubclient/samples/__init__.py create mode 100644 pephubclient/samples/samples.py create mode 100644 pephubclient/views/__init__.py create mode 100644 pephubclient/views/views.py diff --git a/pephubclient/files_manager.py b/pephubclient/files_manager.py index a934186..d3a9693 100644 --- a/pephubclient/files_manager.py +++ b/pephubclient/files_manager.py @@ -48,7 +48,9 @@ def create_project_folder( ) if parent_path: if not Path(parent_path).exists(): - raise OSError(f"Parent path does not exist. Provided path: {parent_path}") + raise OSError( + f"Parent path does not exist. Provided path: {parent_path}" + ) folder_path = os.path.join(parent_path or os.getcwd(), folder_name) Path(folder_path).mkdir(parents=True, exist_ok=True) return folder_path diff --git a/pephubclient/pephubclient.py b/pephubclient/pephubclient.py index 0e8cb34..eaf812a 100644 --- a/pephubclient/pephubclient.py +++ b/pephubclient/pephubclient.py @@ -37,6 +37,8 @@ ProjectAnnotationModel, ) from pephubclient.pephub_oauth.pephub_oauth import PEPHubAuth +from pephubclient.samples import Samples +from pephubclient.views import Views urllib3.disable_warnings() @@ -52,6 +54,16 @@ class PEPHubClient(RequestManager): def __init__(self): self.registry_path = None + self.__view = Views() + self.__sample = Samples() + + @property + def view(self): + return self.__view + + @property + def sample(self): + return self.__sample def login(self) -> NoReturn: """ diff --git a/pephubclient/samples/__init__.py b/pephubclient/samples/__init__.py new file mode 100644 index 0000000..67f2f90 --- /dev/null +++ b/pephubclient/samples/__init__.py @@ -0,0 +1,3 @@ +from samples import Samples + +__all__ = ["Samples"] diff --git a/pephubclient/samples/samples.py b/pephubclient/samples/samples.py new file mode 100644 index 0000000..4e07181 --- /dev/null +++ b/pephubclient/samples/samples.py @@ -0,0 +1,38 @@ +from ..files_manager import FilesManager + + +class Samples: + def __init__(self): + self.jwt_data = "" + + def get( + self, + namespace: str, + name: str, + tag: str, + sample_name: str = None, + ): + ... + + def create( + self, + namespace: str, + name: str, + tag: str, + sample_name: str, + sample_dict: dict, + ): + ... + + def update( + self, + namespace: str, + name: str, + tag: str, + sample_name: str, + sample_dict: dict, + ): + ... + + def remove(self, namespace: str, name: str, tag: str, sample_name: str): + ... diff --git a/pephubclient/views/__init__.py b/pephubclient/views/__init__.py new file mode 100644 index 0000000..4c3c1b4 --- /dev/null +++ b/pephubclient/views/__init__.py @@ -0,0 +1,3 @@ +from views import Views + +__all__ = ["Views"] diff --git a/pephubclient/views/views.py b/pephubclient/views/views.py new file mode 100644 index 0000000..04213e3 --- /dev/null +++ b/pephubclient/views/views.py @@ -0,0 +1,34 @@ +class Views: + def __init__(self, jwt_data: str): + self._jwt_data = jwt_data + + def get(self, namespace: str, name: str, tag: str, view_name: str): + ... + + def create( + self, namespace: str, name: str, tag: str, view_name: str, view_dict: dict + ): + ... + + def delete(self, namespace: str, name: str, tag: str, view_name: str): + ... + + def add_sample( + self, + namespace: str, + name: str, + tag: str, + view_name: str, + sample_name: str, + ): + ... + + def remove_sample( + self, + namespace: str, + name: str, + tag: str, + view_name: str, + sample_name: str, + ): + ... From 72af16a8923d31e02ef8722929d7c207b36b2410 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 23 Jan 2024 17:05:42 -0500 Subject: [PATCH 08/19] updated docstring --- pephubclient/pephubclient.py | 4 ++-- pephubclient/samples/samples.py | 6 ++++++ pephubclient/views/views.py | 9 ++++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/pephubclient/pephubclient.py b/pephubclient/pephubclient.py index eaf812a..2352fc3 100644 --- a/pephubclient/pephubclient.py +++ b/pephubclient/pephubclient.py @@ -58,11 +58,11 @@ def __init__(self): self.__sample = Samples() @property - def view(self): + def view(self) -> Views: return self.__view @property - def sample(self): + def sample(self) -> Samples: return self.__sample def login(self) -> NoReturn: diff --git a/pephubclient/samples/samples.py b/pephubclient/samples/samples.py index 4e07181..09d86e0 100644 --- a/pephubclient/samples/samples.py +++ b/pephubclient/samples/samples.py @@ -2,6 +2,12 @@ class Samples: + """ + Class for managing samples in PEPhub and provides methods for + getting, creating, updating and removing samples. + This class is not related to peppy.Sample class. + """ + def __init__(self): self.jwt_data = "" diff --git a/pephubclient/views/views.py b/pephubclient/views/views.py index 04213e3..dd41597 100644 --- a/pephubclient/views/views.py +++ b/pephubclient/views/views.py @@ -1,5 +1,12 @@ class Views: - def __init__(self, jwt_data: str): + """ + Class for managing views in PEPhub and provides methods for + getting, creating, updating and removing views. + + This class aims to warp the Views API for easier maintenance and + better user experience. + """ + def __init__(self, jwt_data: str = None): self._jwt_data = jwt_data def get(self, namespace: str, name: str, tag: str, view_name: str): From 1bbd3276c64db0b45a3692a6281f086511b1e8bc Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Fri, 26 Jan 2024 14:43:07 -0500 Subject: [PATCH 09/19] updated delete function --- pephubclient/files_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pephubclient/files_manager.py b/pephubclient/files_manager.py index 84bebc5..6331ed8 100644 --- a/pephubclient/files_manager.py +++ b/pephubclient/files_manager.py @@ -69,6 +69,9 @@ def file_exists(full_path: str) -> bool: def delete_file_if_exists(filename: str) -> None: with suppress(FileNotFoundError): os.remove(filename) + print( + f"\033[38;5;11m{f'File was deleted successfully -> {filename}'}\033[0m" + ) @staticmethod def check_writable(path: str, force: bool = True): From 8a7f8a7f6d79130f7ff712dfb41e4548652bc26a Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Sat, 27 Jan 2024 20:12:58 -0500 Subject: [PATCH 10/19] updated changelog --- docs/changelog.md | 8 ++++++-- pephubclient/__init__.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index d6c6474..0a0194b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,11 +2,15 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. -## [0.4.0] - 2024-XX-XX +## [0.3.0] - 2024-XX-XX ### Added - Added param parent dir where peps should be saved +- Added zip option to save_pep function -## [0.3.0] - 2024-01-17 +### Changed +- Transferred save_pep function to helpers + +## [0.2.2] - 2024-01-17 ### Added - customization of the base pephub URL. [#22](https://github.com/pepkit/pephubclient/issues/22) diff --git a/pephubclient/__init__.py b/pephubclient/__init__.py index 39deb45..45fd561 100644 --- a/pephubclient/__init__.py +++ b/pephubclient/__init__.py @@ -2,7 +2,7 @@ from pephubclient.helpers import is_registry_path, save_pep __app_name__ = "pephubclient" -__version__ = "0.4.0" +__version__ = "0.3.0" __author__ = "Oleksandr Khoroshevskyi, Rafal Stepien" From 4229494210506688fbf1d4aea8c3c0c9578cd7da Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 30 Jan 2024 23:14:43 +0100 Subject: [PATCH 11/19] added samples functionality --- MANIFEST.in | 4 +- pephubclient/constants.py | 9 + pephubclient/helpers.py | 28 ++++ pephubclient/modules/__init__.py | 0 pephubclient/modules/sample.py | 155 ++++++++++++++++++ .../{views/views.py => modules/view.py} | 24 +-- pephubclient/pephubclient.py | 85 +++------- pephubclient/samples/__init__.py | 3 - pephubclient/samples/samples.py | 44 ----- pephubclient/views/__init__.py | 3 - tests/test_pephubclient.py | 111 +++++++++---- 11 files changed, 310 insertions(+), 156 deletions(-) create mode 100644 pephubclient/modules/__init__.py create mode 100644 pephubclient/modules/sample.py rename pephubclient/{views/views.py => modules/view.py} (80%) delete mode 100644 pephubclient/samples/__init__.py delete mode 100644 pephubclient/samples/samples.py delete mode 100644 pephubclient/views/__init__.py diff --git a/MANIFEST.in b/MANIFEST.in index 4797e8e..185b75e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,5 @@ include requirements/* include README.md -include pephubclient/pephub_oauth/* \ No newline at end of file +include pephubclient/pephub_oauth/* +include pephubclient/samples/* +include pephubclient/views/* \ No newline at end of file diff --git a/pephubclient/constants.py b/pephubclient/constants.py index d625f58..f2ae032 100644 --- a/pephubclient/constants.py +++ b/pephubclient/constants.py @@ -12,6 +12,8 @@ PEPHUB_PEP_SEARCH_URL = f"{PEPHUB_BASE_URL}api/v1/namespaces/{{namespace}}/projects" PEPHUB_PUSH_URL = f"{PEPHUB_BASE_URL}api/v1/namespaces/{{namespace}}/projects/json" +PEPHUB_SAMPLE_URL = f"{PEPHUB_BASE_URL}api/v1/projects/{{namespace}}/{{project}}/samples/{{sample_name}}" + class RegistryPath(BaseModel): protocol: Optional[str] = None @@ -33,3 +35,10 @@ class ResponseStatusCodes(int, Enum): NOT_EXIST = 404 CONFLICT = 409 INTERNAL_ERROR = 500 + + +USER_DATA_FILE_NAME = "jwt.txt" +HOME_PATH = os.getenv("HOME") +if not HOME_PATH: + HOME_PATH = os.path.expanduser("~") +PATH_TO_FILE_WITH_JWT = os.path.join(HOME_PATH, ".pephubclient/") + USER_DATA_FILE_NAME diff --git a/pephubclient/helpers.py b/pephubclient/helpers.py index 7d582a1..7d376c1 100644 --- a/pephubclient/helpers.py +++ b/pephubclient/helpers.py @@ -61,6 +61,34 @@ def decode_response(response: requests.Response, encoding: str = "utf-8") -> str except json.JSONDecodeError as err: raise ResponseError(f"Error in response encoding format: {err}") + @staticmethod + def parse_query_param(pep_variables: dict) -> str: + """ + Grab all the variables passed by user (if any) and parse them to match the format specified + by PEPhub API for query parameters. + + :param pep_variables: dict of query parameters + :return: PEPHubClient variables transformed into string in correct format. + """ + parsed_variables = [] + + for variable_name, variable_value in pep_variables.items(): + parsed_variables.append(f"{variable_name}={variable_value}") + return "?" + "&".join(parsed_variables) + + @staticmethod + def parse_header(jwt_data: Optional[str] = None) -> dict: + """ + Create Authorization header + + :param jwt_data: jwt string + :return: Authorization dict + """ + if jwt_data: + return {"Authorization": jwt_data} + else: + return {} + class MessageHandler: """ diff --git a/pephubclient/modules/__init__.py b/pephubclient/modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pephubclient/modules/sample.py b/pephubclient/modules/sample.py new file mode 100644 index 0000000..4e2ae5e --- /dev/null +++ b/pephubclient/modules/sample.py @@ -0,0 +1,155 @@ +from pephubclient.helpers import RequestManager +from pephubclient.constants import PEPHUB_SAMPLE_URL +import json + + +class PEPHubSample(RequestManager): + """ + Class for managing samples in PEPhub and provides methods for + getting, creating, updating and removing samples. + This class is not related to peppy.Sample class. + """ + + def __init__(self, jwt_data: str = None): + """ + :param jwt_data: jwt token for authorization + """ + + self.__jwt_data = jwt_data + + def get( + self, + namespace: str, + name: str, + tag: str, + sample_name: str = None, + ) -> dict: + """ + Get sample from project in PEPhub. + + :param namespace: namespace of project + :param name: name of project + :param tag: tag of project + :param sample_name: sample name + :return: Sample object + """ + url = self._build_sample_request_url( + namespace=namespace, name=name, sample_name=sample_name + ) + + url = url + self.parse_query_param(pep_variables={"tag": tag}) + + response = self.send_request( + method="GET", url=url, headers=self.parse_header(self.__jwt_data) + ) + output = dict(json.loads(self.decode_response(response))) + return output + + def create( + self, + namespace: str, + name: str, + tag: str, + sample_name: str, + sample_dict: dict, + overwrite: bool = False, + ) -> None: + """ + Create sample in project in PEPhub. + + :param namespace: namespace of project + :param name: name of project + :param tag: tag of project + :param sample_dict: sample dict + :param sample_name: sample name + :param overwrite: overwrite sample if it exists + :return: None + """ + url = self._build_sample_request_url( + namespace=namespace, + name=name, + sample_name=sample_name, + ) + + url = url + self.parse_query_param( + pep_variables={"tag": tag, "overwrite": overwrite} + ) + + response = self.send_request( + method="POST", + url=url, + headers=self.parse_header(self.__jwt_data), + json=sample_dict, + ) + output = self.decode_response(response) + return output + + def update( + self, + namespace: str, + name: str, + tag: str, + sample_name: str, + sample_dict: dict, + ): + """ + Update sample in project in PEPhub. + + :param namespace: namespace of project + :param name: name of project + :param tag: tag of project + :param sample_name: sample name + :param sample_dict: sample dict, that contain elements to update, or + :return: None + """ + + url = self._build_sample_request_url( + namespace=namespace, name=name, sample_name=sample_name + ) + + url = url + self.parse_query_param(pep_variables={"tag": tag}) + + response = self.send_request( + method="PATCH", + url=url, + headers=self.parse_header(self.__jwt_data), + json=sample_dict, + ) + output = self.decode_response(response) + return output + + def remove(self, namespace: str, name: str, tag: str, sample_name: str): + """ + Remove sample from project in PEPhub. + + :param namespace: namespace of project + :param name: name of project + :param tag: tag of project + :param sample_name: sample name + :return: None + """ + url = self._build_sample_request_url( + namespace=namespace, name=name, sample_name=sample_name + ) + + url = url + self.parse_query_param(pep_variables={"tag": tag}) + + response = self.send_request( + method="DELETE", + url=url, + headers=self.parse_header(self.__jwt_data), + ) + output = self.decode_response(response) + return output + + @staticmethod + def _build_sample_request_url(namespace: str, name: str, sample_name: str) -> str: + """ + Build url for sample request. + + :param namespace: namespace where project will be uploaded + :return: url string + """ + return PEPHUB_SAMPLE_URL.format( + namespace=namespace, project=name, sample_name=sample_name + ) diff --git a/pephubclient/views/views.py b/pephubclient/modules/view.py similarity index 80% rename from pephubclient/views/views.py rename to pephubclient/modules/view.py index dd41597..19d2ed9 100644 --- a/pephubclient/views/views.py +++ b/pephubclient/modules/view.py @@ -1,4 +1,4 @@ -class Views: +class PEPHubView: """ Class for managing views in PEPhub and provides methods for getting, creating, updating and removing views. @@ -6,19 +6,21 @@ class Views: This class aims to warp the Views API for easier maintenance and better user experience. """ + def __init__(self, jwt_data: str = None): - self._jwt_data = jwt_data + """ + :param jwt_data: jwt token for authorization + """ + + self.__jwt_data = jwt_data - def get(self, namespace: str, name: str, tag: str, view_name: str): - ... + def get(self, namespace: str, name: str, tag: str, view_name: str): ... def create( self, namespace: str, name: str, tag: str, view_name: str, view_dict: dict - ): - ... + ): ... - def delete(self, namespace: str, name: str, tag: str, view_name: str): - ... + def delete(self, namespace: str, name: str, tag: str, view_name: str): ... def add_sample( self, @@ -27,8 +29,7 @@ def add_sample( tag: str, view_name: str, sample_name: str, - ): - ... + ): ... def remove_sample( self, @@ -37,5 +38,4 @@ def remove_sample( tag: str, view_name: str, sample_name: str, - ): - ... + ): ... diff --git a/pephubclient/pephubclient.py b/pephubclient/pephubclient.py index 2768a54..1cd62b5 100644 --- a/pephubclient/pephubclient.py +++ b/pephubclient/pephubclient.py @@ -1,5 +1,4 @@ import json -import os from typing import NoReturn, Optional, Literal from typing_extensions import deprecated @@ -16,6 +15,7 @@ RegistryPath, ResponseStatusCodes, PEPHUB_PEP_SEARCH_URL, + PATH_TO_FILE_WITH_JWT, ) from pephubclient.exceptions import ( IncorrectQueryStringError, @@ -30,32 +30,25 @@ ProjectAnnotationModel, ) from pephubclient.pephub_oauth.pephub_oauth import PEPHubAuth -from pephubclient.samples import Samples -from pephubclient.views import Views +from pephubclient.modules.view import PEPHubView +from pephubclient.modules.sample import PEPHubSample urllib3.disable_warnings() class PEPHubClient(RequestManager): - USER_DATA_FILE_NAME = "jwt.txt" - home_path = os.getenv("HOME") - if not home_path: - home_path = os.path.expanduser("~") - PATH_TO_FILE_WITH_JWT = ( - os.path.join(home_path, ".pephubclient/") + USER_DATA_FILE_NAME - ) - def __init__(self): - self.registry_path = None - self.__view = Views() - self.__sample = Samples() + self.__jwt_data = FilesManager.load_jwt_data_from_file(PATH_TO_FILE_WITH_JWT) + + self.__view = PEPHubView(self.__jwt_data) + self.__sample = PEPHubSample(self.__jwt_data) @property - def view(self) -> Views: + def view(self) -> PEPHubView: return self.__view @property - def sample(self) -> Samples: + def sample(self) -> PEPHubSample: return self.__sample def login(self) -> NoReturn: @@ -64,13 +57,15 @@ def login(self) -> NoReturn: """ user_token = PEPHubAuth().login_to_pephub() - FilesManager.save_jwt_data_to_file(self.PATH_TO_FILE_WITH_JWT, user_token) + FilesManager.save_jwt_data_to_file(PATH_TO_FILE_WITH_JWT, user_token) + self.__jwt_data = FilesManager.load_jwt_data_from_file(PATH_TO_FILE_WITH_JWT) def logout(self) -> NoReturn: """ Log out from PEPhub """ - FilesManager.delete_file_if_exists(self.PATH_TO_FILE_WITH_JWT) + FilesManager.delete_file_if_exists(PATH_TO_FILE_WITH_JWT) + self.__jwt_data = None def pull( self, @@ -88,9 +83,8 @@ def pull( :param str output: path where project will be saved :return: None """ - jwt_data = FilesManager.load_jwt_data_from_file(self.PATH_TO_FILE_WITH_JWT) project_dict = self.load_raw_pep( - registry_path=project_registry_path, jwt_data=jwt_data + registry_path=project_registry_path, ) save_pep( @@ -113,8 +107,8 @@ def load_project( :param query_param: query parameters used in get request :return Project: peppy project. """ - jwt = FilesManager.load_jwt_data_from_file(self.PATH_TO_FILE_WITH_JWT) - raw_pep = self.load_raw_pep(project_registry_path, jwt, query_param) + jwt = FilesManager.load_jwt_data_from_file(PATH_TO_FILE_WITH_JWT) + raw_pep = self.load_raw_pep(project_registry_path, query_param) peppy_project = peppy.Project().from_dict(raw_pep) return peppy_project @@ -170,7 +164,6 @@ def upload( :param force: overwrite project if it exists :return: None """ - jwt_data = FilesManager.load_jwt_data_from_file(self.PATH_TO_FILE_WITH_JWT) if name: project[NAME_KEY] = name @@ -186,7 +179,7 @@ def upload( pephub_response = self.send_request( method="POST", url=self._build_push_request_url(namespace=namespace), - headers=self._get_header(jwt_data), + headers=self.parse_header(self.__jwt_data), json=upload_data.model_dump(), cookies=None, ) @@ -232,7 +225,6 @@ def find_project( :param end_date: filter end date (if none today's date is used) :return: """ - jwt_data = FilesManager.load_jwt_data_from_file(self.PATH_TO_FILE_WITH_JWT) query_param = { "q": query_string, @@ -253,7 +245,7 @@ def find_project( pephub_response = self.send_request( method="GET", url=url, - headers=self._get_header(jwt_data), + headers=self.parse_header(self.__jwt_data), json=None, cookies=None, ) @@ -278,15 +270,13 @@ def _load_raw_pep( :param registry_path: Project namespace, eg. "geo/GSE124224:tag" :param query_param: Optional variables to be passed to PEPhub - :param jwt_data: JWT token. :return: Raw project in dict. """ - return self.load_raw_pep(registry_path, jwt_data, query_param) + return self.load_raw_pep(registry_path, query_param) def load_raw_pep( self, registry_path: str, - jwt_data: Optional[str] = None, query_param: Optional[dict] = None, ) -> dict: """ @@ -294,11 +284,8 @@ def load_raw_pep( :param registry_path: Project namespace, eg. "geo/GSE124224:tag" :param query_param: Optional variables to be passed to PEPhub - :param jwt_data: JWT token. :return: Raw project in dict. """ - if not jwt_data: - jwt_data = FilesManager.load_jwt_data_from_file(self.PATH_TO_FILE_WITH_JWT) query_param = query_param or {} query_param["raw"] = "true" @@ -306,7 +293,7 @@ def load_raw_pep( pephub_response = self.send_request( method="GET", url=self._build_pull_request_url(query_param=query_param), - headers=self._get_header(jwt_data), + headers=self.parse_header(self.__jwt_data), cookies=None, ) if pephub_response.status_code == ResponseStatusCodes.OK: @@ -335,19 +322,6 @@ def _set_registry_data(self, query_string: str) -> None: except (ValidationError, TypeError): raise IncorrectQueryStringError(query_string=query_string) - @staticmethod - def _get_header(jwt_data: Optional[str] = None) -> dict: - """ - Create Authorization header - - :param jwt_data: jwt string - :return: Authorization dict - """ - if jwt_data: - return {"Authorization": jwt_data} - else: - return {} - def _build_pull_request_url(self, query_param: dict = None) -> str: """ Build request for getting projects form pephub @@ -360,7 +334,7 @@ def _build_pull_request_url(self, query_param: dict = None) -> str: endpoint = self.registry_path.namespace + "/" + self.registry_path.item - variables_string = PEPHubClient._parse_query_param(query_param) + variables_string = self.parse_query_param(query_param) endpoint += variables_string return PEPHUB_PEP_API_BASE_URL + endpoint @@ -374,7 +348,7 @@ def _build_project_search_url(namespace: str, query_param: dict = None) -> str: :return: url string """ - variables_string = PEPHubClient._parse_query_param(query_param) + variables_string = RequestManager.parse_query_param(query_param) endpoint = variables_string return PEPHUB_PEP_SEARCH_URL.format(namespace=namespace) + endpoint @@ -389,21 +363,6 @@ def _build_push_request_url(namespace: str) -> str: """ return PEPHUB_PUSH_URL.format(namespace=namespace) - @staticmethod - def _parse_query_param(pep_variables: dict) -> str: - """ - Grab all the variables passed by user (if any) and parse them to match the format specified - by PEPhub API for query parameters. - - :param pep_variables: dict of query parameters - :return: PEPHubClient variables transformed into string in correct format. - """ - parsed_variables = [] - - for variable_name, variable_value in pep_variables.items(): - parsed_variables.append(f"{variable_name}={variable_value}") - return "?" + "&".join(parsed_variables) - @staticmethod def _handle_pephub_response(pephub_response: requests.Response): """ diff --git a/pephubclient/samples/__init__.py b/pephubclient/samples/__init__.py deleted file mode 100644 index 67f2f90..0000000 --- a/pephubclient/samples/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from samples import Samples - -__all__ = ["Samples"] diff --git a/pephubclient/samples/samples.py b/pephubclient/samples/samples.py deleted file mode 100644 index 09d86e0..0000000 --- a/pephubclient/samples/samples.py +++ /dev/null @@ -1,44 +0,0 @@ -from ..files_manager import FilesManager - - -class Samples: - """ - Class for managing samples in PEPhub and provides methods for - getting, creating, updating and removing samples. - This class is not related to peppy.Sample class. - """ - - def __init__(self): - self.jwt_data = "" - - def get( - self, - namespace: str, - name: str, - tag: str, - sample_name: str = None, - ): - ... - - def create( - self, - namespace: str, - name: str, - tag: str, - sample_name: str, - sample_dict: dict, - ): - ... - - def update( - self, - namespace: str, - name: str, - tag: str, - sample_name: str, - sample_dict: dict, - ): - ... - - def remove(self, namespace: str, name: str, tag: str, sample_name: str): - ... diff --git a/pephubclient/views/__init__.py b/pephubclient/views/__init__.py deleted file mode 100644 index 4c3c1b4..0000000 --- a/pephubclient/views/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from views import Views - -__all__ = ["Views"] diff --git a/tests/test_pephubclient.py b/tests/test_pephubclient.py index b179b94..234e278 100644 --- a/tests/test_pephubclient.py +++ b/tests/test_pephubclient.py @@ -202,37 +202,88 @@ class TestHelpers: def test_is_registry_path(self, input_str, expected_output): assert is_registry_path(input_str) is expected_output - @pytest.mark.skipif(True, reason="not implemented yet") - def test_save_zip_pep(self): - ... - - @pytest.mark.skipif(True, reason="not implemented yet") - def test_save_unzip_pep(self): - ... - - -@pytest.mark.skipif(True, reason="not implemented yet") -class TestSamplesModification: - def test_get_sumple(self): - ... - - def test_add_sample(self): - ... - - def test_remove_sample(self): - ... - - def test_update_sample(self): - ... +# @pytest.mark.skipif(True, reason="not implemented yet") +# def test_save_zip_pep(self): +# ... +# +# @pytest.mark.skipif(True, reason="not implemented yet") +# def test_save_unzip_pep(self): +# ... +# +# +# @pytest.mark.skipif(True, reason="not implemented yet") +# class TestSamplesModification: +# def test_get_sumple(self): +# ... +# +# def test_add_sample(self): +# ... +# +# def test_remove_sample(self): +# ... +# +# def test_update_sample(self): +# ... +# +# +# @pytest.mark.skipif(True, reason="not implemented yet") +# class TestProjectVeiw: +# def test_get_view(self): +# ... +# +# def test_create_view(self): +# ... +# +# def test_delete_view(self): +# ... +# +# +class TestManual: + def test_manual(self): + ff = PEPHubClient().sample.get( + "khoroshevskyi", + "bedset1", + "default", + "newf", + ) -@pytest.mark.skipif(True, reason="not implemented yet") -class TestProjectVeiw: - def test_get_view(self): - ... + def test_update(self): + ff = PEPHubClient().sample.get( + "khoroshevskyi", + "bedset1", + "default", + "newf", + ) + ff.update({"fff": "test1"}) + ff["sample_type"] = "new_type" + PEPHubClient().sample.update( + "khoroshevskyi", + "bedset1", + "default", + "newf", + sample_dict=ff, + ) - def test_create_view(self): - ... + def test_add(self): + ff = { + "genome": "phc_test1", + "sample_type": "phc_test", + "sample_name": "test_phc", + } + PEPHubClient().sample.create( + "khoroshevskyi", + "bedset1", + "default", + "test_phc", + overwrite=True, + sample_dict=ff, + ) - def test_delete_view(self): - ... + def test_delete(self): + PEPHubClient().sample.remove( + "khoroshevskyi", + "bedset1", + "default", + "test_phc", + ) From a0ea82c1fcc0e7ffa29fe3e92d6e8ae1e75725db Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Thu, 1 Feb 2024 19:38:31 +0100 Subject: [PATCH 12/19] updated manifest --- MANIFEST.in | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 185b75e..a948aa7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ include requirements/* include README.md include pephubclient/pephub_oauth/* -include pephubclient/samples/* -include pephubclient/views/* \ No newline at end of file +include pephubclient/modules/* \ No newline at end of file From dadffc675826136ea14c8e80043c5164f4bbd9bf Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Thu, 1 Feb 2024 22:30:27 +0100 Subject: [PATCH 13/19] work on samples and viewes --- pephubclient/constants.py | 6 ++ pephubclient/helpers.py | 10 +- pephubclient/modules/sample.py | 32 ++++-- pephubclient/modules/view.py | 191 +++++++++++++++++++++++++++++++-- 4 files changed, 220 insertions(+), 19 deletions(-) diff --git a/pephubclient/constants.py b/pephubclient/constants.py index f2ae032..a1a505b 100644 --- a/pephubclient/constants.py +++ b/pephubclient/constants.py @@ -13,6 +13,12 @@ PEPHUB_PUSH_URL = f"{PEPHUB_BASE_URL}api/v1/namespaces/{{namespace}}/projects/json" PEPHUB_SAMPLE_URL = f"{PEPHUB_BASE_URL}api/v1/projects/{{namespace}}/{{project}}/samples/{{sample_name}}" +PEPHUB_VIEW_URL = ( + f"{PEPHUB_BASE_URL}api/v1/projects/{{namespace}}/{{project}}/views/{{view_name}}" +) +PEPHUB_VIEW_SAMPLE_URL = ( + f"{PEPHUB_BASE_URL}api/v1/projects/{{namespace}}/{{project}}/views/{{view_name}}/{{sample_name}}" +) class RegistryPath(BaseModel): diff --git a/pephubclient/helpers.py b/pephubclient/helpers.py index 7d376c1..a9f764b 100644 --- a/pephubclient/helpers.py +++ b/pephubclient/helpers.py @@ -47,17 +47,23 @@ def send_request( ) @staticmethod - def decode_response(response: requests.Response, encoding: str = "utf-8") -> str: + def decode_response( + response: requests.Response, encoding: str = "utf-8", output_json: bool = False + ) -> Union[str, dict]: """ Decode the response from GitHub and pack the returned data into appropriate model. :param response: Response from GitHub. :param encoding: Response encoding [Default: utf-8] + :param output_json: If True, return response in json format :return: Response data as an instance of correct model. """ try: - return response.content.decode(encoding) + if output_json: + return response.json() + else: + return response.content.decode(encoding) except json.JSONDecodeError as err: raise ResponseError(f"Error in response encoding format: {err}") diff --git a/pephubclient/modules/sample.py b/pephubclient/modules/sample.py index 4e2ae5e..68ec9b5 100644 --- a/pephubclient/modules/sample.py +++ b/pephubclient/modules/sample.py @@ -1,6 +1,6 @@ from pephubclient.helpers import RequestManager -from pephubclient.constants import PEPHUB_SAMPLE_URL -import json +from pephubclient.constants import PEPHUB_SAMPLE_URL, ResponseStatusCodes +from pephubclient.exceptions import ResponseError class PEPHubSample(RequestManager): @@ -42,8 +42,8 @@ def get( response = self.send_request( method="GET", url=url, headers=self.parse_header(self.__jwt_data) ) - output = dict(json.loads(self.decode_response(response))) - return output + if response.status_code == ResponseStatusCodes.OK: + return self.decode_response(response, output_json=True) def create( self, @@ -81,8 +81,12 @@ def create( headers=self.parse_header(self.__jwt_data), json=sample_dict, ) - output = self.decode_response(response) - return output + if response.status_code == ResponseStatusCodes.OK: + return None + else: + raise ResponseError( + f"Unexpected return value. Error: {response.status_code}" + ) def update( self, @@ -115,8 +119,12 @@ def update( headers=self.parse_header(self.__jwt_data), json=sample_dict, ) - output = self.decode_response(response) - return output + if response.status_code == ResponseStatusCodes.OK: + return None + else: + raise ResponseError( + f"Unexpected return value. Error: {response.status_code}" + ) def remove(self, namespace: str, name: str, tag: str, sample_name: str): """ @@ -139,8 +147,12 @@ def remove(self, namespace: str, name: str, tag: str, sample_name: str): url=url, headers=self.parse_header(self.__jwt_data), ) - output = self.decode_response(response) - return output + if response.status_code == ResponseStatusCodes.OK: + return None + else: + raise ResponseError( + f"Unexpected return value. Error: {response.status_code}" + ) @staticmethod def _build_sample_request_url(namespace: str, name: str, sample_name: str) -> str: diff --git a/pephubclient/modules/view.py b/pephubclient/modules/view.py index 19d2ed9..63e1f2a 100644 --- a/pephubclient/modules/view.py +++ b/pephubclient/modules/view.py @@ -1,4 +1,12 @@ -class PEPHubView: +from typing import Union +import peppy + +from pephubclient.helpers import RequestManager +from pephubclient.constants import PEPHUB_VIEW_URL, PEPHUB_VIEW_SAMPLE_URL, ResponseStatusCodes +from pephubclient.exceptions import ResponseError + + +class PEPHubView(RequestManager): """ Class for managing views in PEPhub and provides methods for getting, creating, updating and removing views. @@ -14,13 +22,102 @@ def __init__(self, jwt_data: str = None): self.__jwt_data = jwt_data - def get(self, namespace: str, name: str, tag: str, view_name: str): ... + def get( + self, namespace: str, name: str, tag: str, view_name: str, raw: bool = False + ) -> Union[peppy.Project, dict]: + """ + Get view from project in PEPhub. + + :param namespace: namespace of project + :param name: name of project + :param tag: tag of project + :param view_name: name of the view + :param raw: if True, return raw response + :return: peppy.Project object or dictionary of the project (view) + """ + url = self._build_view_request_url( + namespace=namespace, name=name, view_name=view_name + ) + + url = url + self.parse_query_param(pep_variables={"tag": tag}) + + response = self.send_request( + method="GET", url=url, headers=self.parse_header(self.__jwt_data) + ) + if response.status_code == ResponseStatusCodes.OK: + output = self.decode_response(response, output_json=True) + if raw: + return output + return peppy.Project.from_dict(output) def create( - self, namespace: str, name: str, tag: str, view_name: str, view_dict: dict - ): ... + self, + namespace: str, + name: str, + tag: str, + view_name: str, + sample_list: list = None, + ): + """ + Create view in project in PEPhub. + + :param namespace: namespace of project + :param name: name of project + :param tag: tag of project + :param view_name: name of the view + :param sample_list: list of sample names + """ + url = self._build_view_request_url( + namespace=namespace, name=name, view_name=view_name + ) + + url = url + self.parse_query_param(pep_variables={"tag": tag}) - def delete(self, namespace: str, name: str, tag: str, view_name: str): ... + response = self.send_request( + method="POST", + url=url, + headers=self.parse_header(self.__jwt_data), + json=sample_list, + ) + if response.status_code != ResponseStatusCodes.ACCEPTED: + raise ResponseError( + f"Unexpected return value. Error: {response.status_code}" + ) + + def delete(self, namespace: str, name: str, tag: str, view_name: str) -> None: + """ + Delete view from project in PEPhub. + + :param namespace: namespace of project + :param name: name of project + :param tag: tag of project + :param view_name: name of the view + :return: None + """ + url = self._build_view_request_url( + namespace=namespace, name=name, view_name=view_name + ) + + url = url + self.parse_query_param(pep_variables={"tag": tag}) + + response = self.send_request( + method="DELETE", url=url, headers=self.parse_header(self.__jwt_data) + ) + + if response.status_code == ResponseStatusCodes.ACCEPTED: + pass + elif response.status_code == ResponseStatusCodes.NOT_EXIST: + raise ResponseError("File does not exist, or you are unauthorized.") + elif response.status_code == ResponseStatusCodes.INTERNAL_ERROR: + raise ResponseError( + f"Internal server error. Unexpected return value. Error: {response.status_code}" + ) + else: + raise ResponseError( + f"Unexpected return value. Error: {response.status_code}" + ) + + return None def add_sample( self, @@ -29,7 +126,34 @@ def add_sample( tag: str, view_name: str, sample_name: str, - ): ... + ): + """ + Add sample to view in project in PEPhub. + + :param namespace: namespace of project + :param name: name of project + :param tag: tag of project + :param view_name: name of the view + :param sample_name: name of the sample + """ + url = self._build_view_request_url( + namespace=namespace, + name=name, + view_name=view_name, + sample_name=sample_name, + ) + + url = url + self.parse_query_param(pep_variables={"tag": tag}) + + response = self.send_request( + method="POST", + url=url, + headers=self.parse_header(self.__jwt_data), + ) + if response.status_code != ResponseStatusCodes.ACCEPTED: + raise ResponseError( + f"Unexpected return value. Error: {response.status_code}" + ) def remove_sample( self, @@ -38,4 +162,57 @@ def remove_sample( tag: str, view_name: str, sample_name: str, - ): ... + ): + """ + Remove sample from view in project in PEPhub. + + :param namespace: namespace of project + :param name: name of project + :param tag: tag of project + :param view_name: name of the view + :param sample_name: name of the sample + :return: None + """ + url = self._build_view_request_url( + namespace=namespace, + name=name, + view_name=view_name, + sample_name=sample_name, + ) + + url = url + self.parse_query_param(pep_variables={"tag": tag}) + + response = self.send_request( + method="DELETE", + url=url, + headers=self.parse_header(self.__jwt_data), + ) + if response.status_code != ResponseStatusCodes.ACCEPTED: + raise ResponseError( + f"Unexpected return value. Error: {response.status_code}" + ) + + @staticmethod + def _build_view_request_url( + namespace: str, name: str, view_name: str, sample_name: str = None + ): + """ + Build URL for view request. + + :param namespace: namespace of project + :param name: name of project + :param view_name: name of view + :return: URL + """ + if sample_name: + return PEPHUB_VIEW_SAMPLE_URL.format( + namespace=namespace, + project=name, + view_name=view_name, + sample_name=sample_name, + ) + return PEPHUB_VIEW_URL.format( + namespace=namespace, + project=name, + view_name=view_name, + ) From ff6735bdaf158945359d29e0fbc7ff10c71f8638 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 5 Feb 2024 21:31:23 +0100 Subject: [PATCH 14/19] updated code response --- pephubclient/constants.py | 12 +++-- pephubclient/modules/sample.py | 33 ++++++++++++-- pephubclient/modules/view.py | 60 ++++++++++++++++++------- tests/test_pephubclient.py | 80 +++++++++++++++++++++++++++++++--- 4 files changed, 153 insertions(+), 32 deletions(-) diff --git a/pephubclient/constants.py b/pephubclient/constants.py index a1a505b..27f22cb 100644 --- a/pephubclient/constants.py +++ b/pephubclient/constants.py @@ -4,10 +4,10 @@ from pydantic import BaseModel, field_validator -PEPHUB_BASE_URL = os.getenv( - "PEPHUB_BASE_URL", default="https://pephub-api.databio.org/" -) -# PEPHUB_BASE_URL = "http://0.0.0.0:8000/" +# PEPHUB_BASE_URL = os.getenv( +# "PEPHUB_BASE_URL", default="https://pephub-api.databio.org/" +# ) +PEPHUB_BASE_URL = "http://0.0.0.0:8000/" PEPHUB_PEP_API_BASE_URL = f"{PEPHUB_BASE_URL}api/v1/projects/" PEPHUB_PEP_SEARCH_URL = f"{PEPHUB_BASE_URL}api/v1/namespaces/{{namespace}}/projects" PEPHUB_PUSH_URL = f"{PEPHUB_BASE_URL}api/v1/namespaces/{{namespace}}/projects/json" @@ -16,9 +16,7 @@ PEPHUB_VIEW_URL = ( f"{PEPHUB_BASE_URL}api/v1/projects/{{namespace}}/{{project}}/views/{{view_name}}" ) -PEPHUB_VIEW_SAMPLE_URL = ( - f"{PEPHUB_BASE_URL}api/v1/projects/{{namespace}}/{{project}}/views/{{view_name}}/{{sample_name}}" -) +PEPHUB_VIEW_SAMPLE_URL = f"{PEPHUB_BASE_URL}api/v1/projects/{{namespace}}/{{project}}/views/{{view_name}}/{{sample_name}}" class RegistryPath(BaseModel): diff --git a/pephubclient/modules/sample.py b/pephubclient/modules/sample.py index 68ec9b5..881ed6e 100644 --- a/pephubclient/modules/sample.py +++ b/pephubclient/modules/sample.py @@ -42,7 +42,11 @@ def get( response = self.send_request( method="GET", url=url, headers=self.parse_header(self.__jwt_data) ) - if response.status_code == ResponseStatusCodes.OK: + if response.status_code != ResponseStatusCodes.OK: + raise ResponseError( + f"Sample does not exist, or Internal server error occurred." + ) + else: return self.decode_response(response, output_json=True) def create( @@ -75,14 +79,27 @@ def create( pep_variables={"tag": tag, "overwrite": overwrite} ) + # add sample name to sample_dict if it is not there + if sample_name not in sample_dict.values(): + sample_dict["sample_name"] = sample_name + response = self.send_request( method="POST", url=url, headers=self.parse_header(self.__jwt_data), json=sample_dict, ) - if response.status_code == ResponseStatusCodes.OK: + if response.status_code == ResponseStatusCodes.ACCEPTED: return None + + elif response.status_code == ResponseStatusCodes.NOT_EXIST: + raise ResponseError( + f"Project '{namespace}/{name}:{tag}' does not exist. Error: {response.status_code}" + ) + elif response.status_code == ResponseStatusCodes.CONFLICT: + raise ResponseError( + f"Sample '{sample_name}' already exists. Set overwrite to True to overwrite sample." + ) else: raise ResponseError( f"Unexpected return value. Error: {response.status_code}" @@ -119,8 +136,12 @@ def update( headers=self.parse_header(self.__jwt_data), json=sample_dict, ) - if response.status_code == ResponseStatusCodes.OK: + if response.status_code == ResponseStatusCodes.ACCEPTED: return None + elif response.status_code == ResponseStatusCodes.NOT_EXIST: + raise ResponseError( + f"Sample '{sample_name}' or project {namespace}/{name}:{tag} does not exist. Error: {response.status_code}" + ) else: raise ResponseError( f"Unexpected return value. Error: {response.status_code}" @@ -147,8 +168,12 @@ def remove(self, namespace: str, name: str, tag: str, sample_name: str): url=url, headers=self.parse_header(self.__jwt_data), ) - if response.status_code == ResponseStatusCodes.OK: + if response.status_code == ResponseStatusCodes.ACCEPTED: return None + elif response.status_code == ResponseStatusCodes.NOT_EXIST: + raise ResponseError( + f"Sample '{sample_name}' or project {namespace}/{name}:{tag} does not exist. Error: {response.status_code}" + ) else: raise ResponseError( f"Unexpected return value. Error: {response.status_code}" diff --git a/pephubclient/modules/view.py b/pephubclient/modules/view.py index 63e1f2a..ef14d55 100644 --- a/pephubclient/modules/view.py +++ b/pephubclient/modules/view.py @@ -2,7 +2,11 @@ import peppy from pephubclient.helpers import RequestManager -from pephubclient.constants import PEPHUB_VIEW_URL, PEPHUB_VIEW_SAMPLE_URL, ResponseStatusCodes +from pephubclient.constants import ( + PEPHUB_VIEW_URL, + PEPHUB_VIEW_SAMPLE_URL, + ResponseStatusCodes, +) from pephubclient.exceptions import ResponseError @@ -49,6 +53,12 @@ def get( if raw: return output return peppy.Project.from_dict(output) + elif response.status_code == ResponseStatusCodes.NOT_EXIST: + raise ResponseError("View does not exist, or you are unauthorized.") + else: + raise ResponseError( + f"Internal server error. Unexpected return value. Error: {response.status_code}" + ) def create( self, @@ -79,10 +89,16 @@ def create( headers=self.parse_header(self.__jwt_data), json=sample_list, ) - if response.status_code != ResponseStatusCodes.ACCEPTED: + if response.status_code == ResponseStatusCodes.ACCEPTED: + return None + elif response.status_code == ResponseStatusCodes.NOT_EXIST: raise ResponseError( - f"Unexpected return value. Error: {response.status_code}" + f"Project '{namespace}/{name}:{tag}' or one of the samples does not exist." ) + elif response.status_code == ResponseStatusCodes.CONFLICT: + raise ResponseError(f"View '{view_name}' already exists in the project.") + else: + raise ResponseError(f"Unexpected return value.{response.status_code}") def delete(self, namespace: str, name: str, tag: str, view_name: str) -> None: """ @@ -105,19 +121,13 @@ def delete(self, namespace: str, name: str, tag: str, view_name: str) -> None: ) if response.status_code == ResponseStatusCodes.ACCEPTED: - pass + return None elif response.status_code == ResponseStatusCodes.NOT_EXIST: - raise ResponseError("File does not exist, or you are unauthorized.") - elif response.status_code == ResponseStatusCodes.INTERNAL_ERROR: - raise ResponseError( - f"Internal server error. Unexpected return value. Error: {response.status_code}" - ) + raise ResponseError("View does not exists, or you are unauthorized.") + elif response.status_code == ResponseStatusCodes.UNAUTHORIZED: + raise ResponseError("You are unauthorized to delete this view.") else: - raise ResponseError( - f"Unexpected return value. Error: {response.status_code}" - ) - - return None + raise ResponseError("Unexpected return value. ") def add_sample( self, @@ -150,7 +160,15 @@ def add_sample( url=url, headers=self.parse_header(self.__jwt_data), ) - if response.status_code != ResponseStatusCodes.ACCEPTED: + if response.status_code == ResponseStatusCodes.ACCEPTED: + return None + elif response.status_code == ResponseStatusCodes.NOT_EXIST: + raise ResponseError( + f"Sample '{sample_name}' or project {namespace}/{name}:{tag} does not exist." + ) + elif response.status_code == ResponseStatusCodes.CONFLICT: + raise ResponseError(f"Sample '{sample_name}' already exists in the view.") + else: raise ResponseError( f"Unexpected return value. Error: {response.status_code}" ) @@ -187,7 +205,17 @@ def remove_sample( url=url, headers=self.parse_header(self.__jwt_data), ) - if response.status_code != ResponseStatusCodes.ACCEPTED: + if response.status_code == ResponseStatusCodes.ACCEPTED: + return None + elif response.status_code == ResponseStatusCodes.NOT_EXIST: + raise ResponseError( + f"Sample '{sample_name}' or project {namespace}/{name}:{tag} does not exist. " + ) + elif response.status_code == ResponseStatusCodes.UNAUTHORIZED: + raise ResponseError( + f"You are unauthorized to remove this sample from the view." + ) + else: raise ResponseError( f"Unexpected return value. Error: {response.status_code}" ) diff --git a/tests/test_pephubclient.py b/tests/test_pephubclient.py index 234e278..f6caa53 100644 --- a/tests/test_pephubclient.py +++ b/tests/test_pephubclient.py @@ -245,8 +245,9 @@ def test_manual(self): "khoroshevskyi", "bedset1", "default", - "newf", + "grape1", ) + ff def test_update(self): ff = PEPHubClient().sample.get( @@ -269,14 +270,13 @@ def test_add(self): ff = { "genome": "phc_test1", "sample_type": "phc_test", - "sample_name": "test_phc", } PEPHubClient().sample.create( "khoroshevskyi", "bedset1", "default", - "test_phc", - overwrite=True, + "new_f", + overwrite=False, sample_dict=ff, ) @@ -285,5 +285,75 @@ def test_delete(self): "khoroshevskyi", "bedset1", "default", - "test_phc", + "new_f", + ) + + # test add sample: + # 1. add correct 202 + # 2. add existing 409 + # 3. add with sample_name + # 4. add without sample_name + # 5. add with overwrite + # 6. add to unexisting project 404 + + # delete sample: + # 1. delete existing 202 + # 2. delete unexisting 404 + + # get sample: + # 1. get existing 200 + # 2. get unexisting 404 + # 3. get with raw 200 + # 4. get from unexisting project 404 + + # update sample: + # 1. update existing 202 + # 2. update unexisting sample 404 + # 3. update unexisting project 404 + + +class TestViews: + + def test_get(self): + ff = PEPHubClient().view.get( + "khoroshevskyi", + "bedset1", + "default", + "test_view", + ) + print(ff) + + def test_create(self): + PEPHubClient().view.create( + "khoroshevskyi", + "bedset1", + "default", + "test_view", + sample_list=["orange", "grape1", "apple1"], + ) + + def test_delete(self): + PEPHubClient().view.delete( + "khoroshevskyi", + "bedset1", + "default", + "test_view", + ) + + def test_add_sample(self): + PEPHubClient().view.add_sample( + "khoroshevskyi", + "bedset1", + "default", + "test_view", + "apple", + ) + + def test_delete_sample(self): + PEPHubClient().view.remove_sample( + "khoroshevskyi", + "bedset1", + "default", + "test_view", + "apple", ) From c9fe4617c6315d71fc06e2f71dd4186463b81ab1 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 12 Feb 2024 20:14:53 +0100 Subject: [PATCH 15/19] Added test and updated request methods --- .github/workflows/pytest-windows.yml | 2 +- .github/workflows/pytest.yml | 2 +- Makefile | 2 +- pephubclient/constants.py | 8 +- pephubclient/files_manager.py | 1 - pephubclient/models.py | 5 +- pephubclient/modules/sample.py | 16 +- pephubclient/modules/view.py | 4 +- pephubclient/pephubclient.py | 25 +- requirements/requirements-dev.txt | 0 setup.py | 1 + tests/conftest.py | 4 +- tests/test_manual.py | 101 ++++++ tests/test_pephubclient.py | 489 ++++++++++++++++++++------- 14 files changed, 502 insertions(+), 158 deletions(-) delete mode 100644 requirements/requirements-dev.txt create mode 100644 tests/test_manual.py diff --git a/.github/workflows/pytest-windows.yml b/.github/workflows/pytest-windows.yml index 6884749..34a557e 100644 --- a/.github/workflows/pytest-windows.yml +++ b/.github/workflows/pytest-windows.yml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.10"] + python-version: ["3.11"] os: [windows-latest] steps: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 3a32dfc..334600e 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.8", "3.11"] + python-version: ["3.8", "3.12"] os: [ubuntu-20.04] steps: diff --git a/Makefile b/Makefile index 5033ebb..40d1dfe 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ run-coverage: coverage run -m pytest html-report: - coverage html + coverage html --omit="*/test*" open-coverage: cd htmlcov && google-chrome index.html diff --git a/pephubclient/constants.py b/pephubclient/constants.py index 27f22cb..26e8ed7 100644 --- a/pephubclient/constants.py +++ b/pephubclient/constants.py @@ -4,10 +4,10 @@ from pydantic import BaseModel, field_validator -# PEPHUB_BASE_URL = os.getenv( -# "PEPHUB_BASE_URL", default="https://pephub-api.databio.org/" -# ) -PEPHUB_BASE_URL = "http://0.0.0.0:8000/" +PEPHUB_BASE_URL = os.getenv( + "PEPHUB_BASE_URL", default="https://pephub-api.databio.org/" +) +# PEPHUB_BASE_URL = "http://0.0.0.0:8000/" PEPHUB_PEP_API_BASE_URL = f"{PEPHUB_BASE_URL}api/v1/projects/" PEPHUB_PEP_SEARCH_URL = f"{PEPHUB_BASE_URL}api/v1/namespaces/{{namespace}}/projects" PEPHUB_PUSH_URL = f"{PEPHUB_BASE_URL}api/v1/namespaces/{{namespace}}/projects/json" diff --git a/pephubclient/files_manager.py b/pephubclient/files_manager.py index 6331ed8..a3d9b56 100644 --- a/pephubclient/files_manager.py +++ b/pephubclient/files_manager.py @@ -6,7 +6,6 @@ import yaml import zipfile -from pephubclient.constants import RegistryPath from pephubclient.exceptions import PEPExistsError diff --git a/pephubclient/models.py b/pephubclient/models.py index b4a0172..2df7681 100644 --- a/pephubclient/models.py +++ b/pephubclient/models.py @@ -1,5 +1,5 @@ import datetime -from typing import Optional, List +from typing import Optional, List, Union from pydantic import BaseModel, Field, field_validator, ConfigDict from peppy.const import CONFIG_KEY, SUBSAMPLE_RAW_LIST_KEY, SAMPLE_RAW_DICT_KEY @@ -43,6 +43,9 @@ class ProjectAnnotationModel(BaseModel): submission_date: datetime.datetime digest: str pep_schema: str + pop: bool = False + stars_number: Optional[int] = 0 + forked_from: Optional[Union[str, None]] = None class SearchReturnModel(BaseModel): diff --git a/pephubclient/modules/sample.py b/pephubclient/modules/sample.py index 881ed6e..d66a8f1 100644 --- a/pephubclient/modules/sample.py +++ b/pephubclient/modules/sample.py @@ -42,12 +42,16 @@ def get( response = self.send_request( method="GET", url=url, headers=self.parse_header(self.__jwt_data) ) - if response.status_code != ResponseStatusCodes.OK: + if response.status_code == ResponseStatusCodes.OK: + return self.decode_response(response, output_json=True) + if response.status_code == ResponseStatusCodes.NOT_EXIST: + raise ResponseError("Sample does not exist.") + elif response.status_code == ResponseStatusCodes.INTERNAL_ERROR: + raise ResponseError("Internal server error. Unexpected return value.") + else: raise ResponseError( - f"Sample does not exist, or Internal server error occurred." + f"Unexpected return value. Error: {response.status_code}" ) - else: - return self.decode_response(response, output_json=True) def create( self, @@ -93,9 +97,7 @@ def create( return None elif response.status_code == ResponseStatusCodes.NOT_EXIST: - raise ResponseError( - f"Project '{namespace}/{name}:{tag}' does not exist. Error: {response.status_code}" - ) + raise ResponseError(f"Project '{namespace}/{name}:{tag}' does not exist.") elif response.status_code == ResponseStatusCodes.CONFLICT: raise ResponseError( f"Sample '{sample_name}' already exists. Set overwrite to True to overwrite sample." diff --git a/pephubclient/modules/view.py b/pephubclient/modules/view.py index ef14d55..5633e28 100644 --- a/pephubclient/modules/view.py +++ b/pephubclient/modules/view.py @@ -8,6 +8,7 @@ ResponseStatusCodes, ) from pephubclient.exceptions import ResponseError +from pephubclient.models import ProjectDict class PEPHubView(RequestManager): @@ -52,6 +53,7 @@ def get( output = self.decode_response(response, output_json=True) if raw: return output + output = ProjectDict(**output).model_dump(by_alias=True) return peppy.Project.from_dict(output) elif response.status_code == ResponseStatusCodes.NOT_EXIST: raise ResponseError("View does not exist, or you are unauthorized.") @@ -213,7 +215,7 @@ def remove_sample( ) elif response.status_code == ResponseStatusCodes.UNAUTHORIZED: raise ResponseError( - f"You are unauthorized to remove this sample from the view." + "You are unauthorized to remove this sample from the view." ) else: raise ResponseError( diff --git a/pephubclient/pephubclient.py b/pephubclient/pephubclient.py index 1cd62b5..6aa54ed 100644 --- a/pephubclient/pephubclient.py +++ b/pephubclient/pephubclient.py @@ -1,10 +1,8 @@ -import json from typing import NoReturn, Optional, Literal from typing_extensions import deprecated import peppy from peppy.const import NAME_KEY -import requests import urllib3 from pydantic import ValidationError from ubiquerg import parse_registry_path @@ -107,7 +105,6 @@ def load_project( :param query_param: query parameters used in get request :return Project: peppy project. """ - jwt = FilesManager.load_jwt_data_from_file(PATH_TO_FILE_WITH_JWT) raw_pep = self.load_raw_pep(project_registry_path, query_param) peppy_project = peppy.Project().from_dict(raw_pep) return peppy_project @@ -250,11 +247,11 @@ def find_project( cookies=None, ) if pephub_response.status_code == ResponseStatusCodes.OK: - decoded_response = self._handle_pephub_response(pephub_response) + decoded_response = self.decode_response(pephub_response, output_json=True) project_list = [] - for project_found in json.loads(decoded_response)["items"]: + for project_found in decoded_response["items"]: project_list.append(ProjectAnnotationModel(**project_found)) - return SearchReturnModel(**json.loads(decoded_response)) + return SearchReturnModel(**decoded_response) @deprecated("This method is deprecated. Use load_raw_pep instead.") def _load_raw_pep( @@ -297,8 +294,8 @@ def load_raw_pep( cookies=None, ) if pephub_response.status_code == ResponseStatusCodes.OK: - decoded_response = self._handle_pephub_response(pephub_response) - correct_proj_dict = ProjectDict(**json.loads(decoded_response)) + decoded_response = self.decode_response(pephub_response, output_json=True) + correct_proj_dict = ProjectDict(**decoded_response) # This step is necessary because of this issue: https://github.com/pepkit/pephub/issues/124 return correct_proj_dict.model_dump(by_alias=True) @@ -362,15 +359,3 @@ def _build_push_request_url(namespace: str) -> str: :return: url string """ return PEPHUB_PUSH_URL.format(namespace=namespace) - - @staticmethod - def _handle_pephub_response(pephub_response: requests.Response): - """ - Check pephub response - """ - decoded_response = PEPHubClient.decode_response(pephub_response) - - if pephub_response.status_code != ResponseStatusCodes.OK: - raise ResponseError(message=json.loads(decoded_response).get("detail")) - - return decoded_response diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt deleted file mode 100644 index e69de29..0000000 diff --git a/setup.py b/setup.py index aaf8893..9943860 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ def read_reqs(reqs_name): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering :: Bio-Informatics", ], keywords="project, bioinformatics, metadata", diff --git a/tests/conftest.py b/tests/conftest.py index 48b4483..e0a5469 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,3 @@ -import json - import pytest from pephubclient.pephub_oauth.models import InitializeDeviceCodeResponse @@ -29,7 +27,7 @@ def test_raw_pep_return(): {"time": "0", "file_path": "source1", "sample_name": "frog_0h"}, ], } - return json.dumps(sample_prj) + return sample_prj @pytest.fixture diff --git a/tests/test_manual.py b/tests/test_manual.py new file mode 100644 index 0000000..1848210 --- /dev/null +++ b/tests/test_manual.py @@ -0,0 +1,101 @@ +from pephubclient.pephubclient import PEPHubClient +import pytest + + +@pytest.mark.skip(reason="Manual test") +class TestViewsManual: + + def test_get(self): + ff = PEPHubClient().view.get( + "databio", + "bedset1", + "default", + "test_view", + ) + print(ff) + + def test_create(self): + PEPHubClient().view.create( + "databio", + "bedset1", + "default", + "test_view", + sample_list=["orange", "grape1", "apple1"], + ) + + def test_delete(self): + PEPHubClient().view.delete( + "databio", + "bedset1", + "default", + "test_view", + ) + + def test_add_sample(self): + PEPHubClient().view.add_sample( + "databio", + "bedset1", + "default", + "test_view", + "name", + ) + + def test_delete_sample(self): + PEPHubClient().view.remove_sample( + "databio", + "bedset1", + "default", + "test_view", + "name", + ) + + +@pytest.mark.skip(reason="Manual test") +class TestSamplesManual: + def test_manual(self): + ff = PEPHubClient().sample.get( + "databio", + "bedset1", + "default", + "grape1", + ) + ff + + def test_update(self): + ff = PEPHubClient().sample.get( + "databio", + "bedset1", + "default", + "newf", + ) + ff.update({"shefflab": "test1"}) + ff["sample_type"] = "new_type" + PEPHubClient().sample.update( + "databio", + "bedset1", + "default", + "newf", + sample_dict=ff, + ) + + def test_add(self): + ff = { + "genome": "phc_test1", + "sample_type": "phc_test", + } + PEPHubClient().sample.create( + "databio", + "bedset1", + "default", + "new_2222", + overwrite=False, + sample_dict=ff, + ) + + def test_delete(self): + PEPHubClient().sample.remove( + "databio", + "bedset1", + "default", + "new_2222", + ) diff --git a/tests/test_pephubclient.py b/tests/test_pephubclient.py index f6caa53..6a9aec9 100644 --- a/tests/test_pephubclient.py +++ b/tests/test_pephubclient.py @@ -61,7 +61,7 @@ def test_pull(self, mocker, test_jwt, test_raw_pep_return): return_value=Mock(content="some return", status_code=200), ) mocker.patch( - "pephubclient.pephubclient.PEPHubClient._handle_pephub_response", + "pephubclient.helpers.RequestManager.decode_response", return_value=test_raw_pep_return, ) save_yaml_mock = mocker.patch( @@ -150,11 +150,38 @@ def test_push_with_pephub_error_response( ) def test_search_prj(self, mocker): - return_value = b'{"count":1,"limit":100,"offset":0,"items":[{"namespace":"namespace1","name":"basic","tag":"default","is_private":false,"number_of_samples":2,"description":"None","last_update_date":"2023-08-27 19:07:31.552861+00:00","submission_date":"2023-08-27 19:07:31.552858+00:00","digest":"08cbcdbf4974fc84bee824c562b324b5","pep_schema":"random_schema_name"}],"session_info":null,"can_edit":false}' + return_value = { + "count": 1, + "limit": 100, + "offset": 0, + "items": [ + { + "namespace": "namespace1", + "name": "basic", + "tag": "default", + "is_private": False, + "number_of_samples": 2, + "description": "None", + "last_update_date": "2023-08-27 19:07:31.552861+00:00", + "submission_date": "2023-08-27 19:07:31.552858+00:00", + "digest": "08cbcdbf4974fc84bee824c562b324b5", + "pep_schema": "random_schema_name", + "pop": False, + "stars_number": 0, + "forked_from": None, + } + ], + "session_info": None, + "can_edit": False, + } mocker.patch( "requests.request", return_value=Mock(content=return_value, status_code=200), ) + mocker.patch( + "pephubclient.helpers.RequestManager.decode_response", + return_value=return_value, + ) return_value = PEPHubClient().find_project(namespace="namespace1") assert return_value.count == 1 @@ -203,157 +230,383 @@ def test_is_registry_path(self, input_str, expected_output): assert is_registry_path(input_str) is expected_output -# @pytest.mark.skipif(True, reason="not implemented yet") -# def test_save_zip_pep(self): -# ... -# -# @pytest.mark.skipif(True, reason="not implemented yet") -# def test_save_unzip_pep(self): -# ... -# -# -# @pytest.mark.skipif(True, reason="not implemented yet") -# class TestSamplesModification: -# def test_get_sumple(self): -# ... -# -# def test_add_sample(self): -# ... -# -# def test_remove_sample(self): -# ... -# -# def test_update_sample(self): -# ... -# -# -# @pytest.mark.skipif(True, reason="not implemented yet") -# class TestProjectVeiw: -# def test_get_view(self): -# ... -# -# def test_create_view(self): -# ... -# -# def test_delete_view(self): -# ... -# -# -class TestManual: - def test_manual(self): - ff = PEPHubClient().sample.get( - "khoroshevskyi", - "bedset1", - "default", - "grape1", - ) - ff +class TestSamples: - def test_update(self): - ff = PEPHubClient().sample.get( - "khoroshevskyi", - "bedset1", - "default", - "newf", + def test_get(self, mocker): + return_value = { + "genome": "phc_test1", + "sample_type": "phc_test", + "sample_name": "gg1", + } + mocker.patch( + "requests.request", + return_value=Mock(content=return_value, status_code=200), ) - ff.update({"fff": "test1"}) - ff["sample_type"] = "new_type" - PEPHubClient().sample.update( - "khoroshevskyi", - "bedset1", + mocker.patch( + "pephubclient.helpers.RequestManager.decode_response", + return_value=return_value, + ) + return_value = PEPHubClient().sample.get( + "test_namespace", + "taest_name", "default", - "newf", - sample_dict=ff, + "gg1", + ) + assert return_value == return_value + + @pytest.mark.parametrize( + "status_code, expected_error_message", + [ + ( + 404, + "Sample does not exist.", + ), + ( + 500, + "Internal server error. Unexpected return value.", + ), + ( + 403, + "Unexpected return value. Error: 403", + ), + ], + ) + def test_sample_get_with_pephub_error_response( + self, mocker, status_code, expected_error_message + ): + mocker.patch("requests.request", return_value=Mock(status_code=status_code)) + with pytest.raises(ResponseError, match=expected_error_message): + PEPHubClient().sample.get( + "test_namespace", + "taest_name", + "default", + "gg1", + ) + + @pytest.mark.parametrize( + "prj_dict", + [ + {"genome": "phc_test1", "sample_type": "phc_test", "sample_name": "gg1"}, + {"genome": "phc_test1", "sample_type": "phc_test"}, + ], + ) + def test_create(self, mocker, prj_dict): + return_value = prj_dict + mocker_obj = mocker.patch( + "requests.request", + return_value=Mock(content=return_value, status_code=202), ) - def test_add(self): - ff = { - "genome": "phc_test1", - "sample_type": "phc_test", - } PEPHubClient().sample.create( - "khoroshevskyi", - "bedset1", + "test_namespace", + "taest_name", "default", - "new_f", - overwrite=False, - sample_dict=ff, + "gg1", + sample_dict=return_value, + ) + assert mocker_obj.called + + @pytest.mark.parametrize( + "status_code, expected_error_message", + [ + ( + 404, + "does not exist.", + ), + ( + 409, + "already exists. Set overwrite to True to overwrite sample.", + ), + ( + 500, + "Unexpected return value.", + ), + ], + ) + def test_sample_create_with_pephub_error_response( + self, mocker, status_code, expected_error_message + ): + mocker.patch("requests.request", return_value=Mock(status_code=status_code)) + with pytest.raises(ResponseError, match=expected_error_message): + PEPHubClient().sample.create( + "test_namespace", + "taest_name", + "default", + "gg1", + sample_dict={ + "genome": "phc_test1", + "sample_type": "phc_test", + "sample_name": "gg1", + }, + ) + + def test_delete(self, mocker): + mocker_obj = mocker.patch( + "requests.request", + return_value=Mock(status_code=202), ) - def test_delete(self): PEPHubClient().sample.remove( - "khoroshevskyi", - "bedset1", + "test_namespace", + "taest_name", "default", - "new_f", + "gg1", ) + assert mocker_obj.called - # test add sample: - # 1. add correct 202 - # 2. add existing 409 - # 3. add with sample_name - # 4. add without sample_name - # 5. add with overwrite - # 6. add to unexisting project 404 + @pytest.mark.parametrize( + "status_code, expected_error_message", + [ + ( + 404, + "does not exist.", + ), + ( + 500, + "Unexpected return value.", + ), + ], + ) + def test_sample_delete_with_pephub_error_response( + self, mocker, status_code, expected_error_message + ): + mocker.patch("requests.request", return_value=Mock(status_code=status_code)) + with pytest.raises(ResponseError, match=expected_error_message): + PEPHubClient().sample.remove( + "test_namespace", + "taest_name", + "default", + "gg1", + ) - # delete sample: - # 1. delete existing 202 - # 2. delete unexisting 404 + def test_update(self, mocker): + mocker_obj = mocker.patch( + "requests.request", + return_value=Mock(status_code=202), + ) - # get sample: - # 1. get existing 200 - # 2. get unexisting 404 - # 3. get with raw 200 - # 4. get from unexisting project 404 + PEPHubClient().sample.update( + "test_namespace", + "taest_name", + "default", + "gg1", + sample_dict={ + "genome": "phc_test1", + "sample_type": "phc_test", + "new_col": "column", + }, + ) + assert mocker_obj.called - # update sample: - # 1. update existing 202 - # 2. update unexisting sample 404 - # 3. update unexisting project 404 + @pytest.mark.parametrize( + "status_code, expected_error_message", + [ + ( + 404, + "does not exist.", + ), + ( + 500, + "Unexpected return value.", + ), + ], + ) + def test_sample_update_with_pephub_error_response( + self, mocker, status_code, expected_error_message + ): + mocker.patch("requests.request", return_value=Mock(status_code=status_code)) + with pytest.raises(ResponseError, match=expected_error_message): + PEPHubClient().sample.update( + "test_namespace", + "taest_name", + "default", + "gg1", + sample_dict={ + "genome": "phc_test1", + "sample_type": "phc_test", + "new_col": "column", + }, + ) class TestViews: + def test_get(self, mocker, test_raw_pep_return): + return_value = test_raw_pep_return + mocker.patch( + "requests.request", + return_value=Mock(content=return_value, status_code=200), + ) + mocker.patch( + "pephubclient.helpers.RequestManager.decode_response", + return_value=return_value, + ) - def test_get(self): - ff = PEPHubClient().view.get( - "khoroshevskyi", - "bedset1", + return_value = PEPHubClient().view.get( + "test_namespace", + "taest_name", "default", - "test_view", + "gg1", + ) + assert return_value == return_value + + @pytest.mark.parametrize( + "status_code, expected_error_message", + [ + ( + 404, + "does not exist.", + ), + ( + 500, + "Internal server error.", + ), + ], + ) + def test_view_get_with_pephub_error_response( + self, mocker, status_code, expected_error_message + ): + mocker.patch("requests.request", return_value=Mock(status_code=status_code)) + with pytest.raises(ResponseError, match=expected_error_message): + PEPHubClient().view.get( + "test_namespace", + "taest_name", + "default", + "gg1", + ) + + def test_create(self, mocker): + mocker_obj = mocker.patch( + "requests.request", + return_value=Mock(status_code=202), ) - print(ff) - def test_create(self): PEPHubClient().view.create( - "khoroshevskyi", - "bedset1", + "test_namespace", + "taest_name", "default", - "test_view", - sample_list=["orange", "grape1", "apple1"], + "gg1", + sample_list=["sample1", "sample2"], + ) + assert mocker_obj.called + + @pytest.mark.parametrize( + "status_code, expected_error_message", + [ + ( + 404, + "does not exist.", + ), + ( + 409, + "already exists in the project.", + ), + ], + ) + def test_view_create_with_pephub_error_response( + self, mocker, status_code, expected_error_message + ): + mocker.patch("requests.request", return_value=Mock(status_code=status_code)) + with pytest.raises(ResponseError, match=expected_error_message): + PEPHubClient().view.create( + "test_namespace", + "taest_name", + "default", + "gg1", + sample_list=["sample1", "sample2"], + ) + + def test_delete(self, mocker): + mocker_obj = mocker.patch( + "requests.request", + return_value=Mock(status_code=202), ) - def test_delete(self): PEPHubClient().view.delete( - "khoroshevskyi", - "bedset1", + "test_namespace", + "taest_name", "default", - "test_view", + "gg1", + ) + assert mocker_obj.called + + @pytest.mark.parametrize( + "status_code, expected_error_message", + [ + ( + 404, + "does not exist.", + ), + ( + 401, + "You are unauthorized to delete this view.", + ), + ], + ) + def test_view_delete_with_pephub_error_response( + self, mocker, status_code, expected_error_message + ): + mocker.patch("requests.request", return_value=Mock(status_code=status_code)) + with pytest.raises(ResponseError, match=expected_error_message): + PEPHubClient().view.delete( + "test_namespace", + "taest_name", + "default", + "gg1", + ) + + def test_add_sample(self, mocker): + mocker_obj = mocker.patch( + "requests.request", + return_value=Mock(status_code=202), ) - def test_add_sample(self): PEPHubClient().view.add_sample( - "khoroshevskyi", - "bedset1", + "test_namespace", + "taest_name", "default", - "test_view", - "apple", + "gg1", + "sample1", + ) + assert mocker_obj.called + + def test_delete_sample(self, mocker): + mocker_obj = mocker.patch( + "requests.request", + return_value=Mock(status_code=202), ) - def test_delete_sample(self): PEPHubClient().view.remove_sample( - "khoroshevskyi", - "bedset1", + "test_namespace", + "taest_name", "default", - "test_view", - "apple", + "gg1", + "sample1", ) + assert mocker_obj.called + + +### + + +# test add sample: +# 1. add correct 202 +# 2. add existing 409 +# 3. add with sample_name +# 4. add without sample_name +# 5. add with overwrite +# 6. add to unexisting project 404 + +# delete sample: +# 1. delete existing 202 +# 2. delete unexisting 404 + +# get sample: +# 1. get existing 200 +# 2. get unexisting 404 +# 3. get with raw 200 +# 4. get from unexisting project 404 + +# update sample: +# 1. update existing 202 +# 2. update unexisting sample 404 +# 3. update unexisting project 404 From 0c2cef2a05e8c45a24431137767bf793156358b8 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 12 Feb 2024 20:33:49 +0100 Subject: [PATCH 16/19] Added logging --- pephubclient/__init__.py | 10 ++++++++++ pephubclient/modules/sample.py | 14 +++++++++++++- pephubclient/modules/view.py | 15 +++++++++++++++ requirements/requirements-all.txt | 5 +++-- 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/pephubclient/__init__.py b/pephubclient/__init__.py index 45fd561..3200620 100644 --- a/pephubclient/__init__.py +++ b/pephubclient/__init__.py @@ -1,5 +1,7 @@ from pephubclient.pephubclient import PEPHubClient from pephubclient.helpers import is_registry_path, save_pep +import logging +import coloredlogs __app_name__ = "pephubclient" __version__ = "0.3.0" @@ -14,3 +16,11 @@ "is_registry_path", "save_pep", ] + + +_LOGGER = logging.getLogger(__app_name__) +coloredlogs.install( + logger=_LOGGER, + datefmt="%H:%M:%S", + fmt="[%(levelname)s] [%(asctime)s] %(message)s", +) diff --git a/pephubclient/modules/sample.py b/pephubclient/modules/sample.py index d66a8f1..0194af9 100644 --- a/pephubclient/modules/sample.py +++ b/pephubclient/modules/sample.py @@ -1,7 +1,11 @@ +import logging + from pephubclient.helpers import RequestManager from pephubclient.constants import PEPHUB_SAMPLE_URL, ResponseStatusCodes from pephubclient.exceptions import ResponseError +_LOGGER = logging.getLogger("pephubclient") + class PEPHubSample(RequestManager): """ @@ -94,8 +98,10 @@ def create( json=sample_dict, ) if response.status_code == ResponseStatusCodes.ACCEPTED: + _LOGGER.info( + f"Sample '{sample_name}' added to project '{namespace}/{name}:{tag}' successfully." + ) return None - elif response.status_code == ResponseStatusCodes.NOT_EXIST: raise ResponseError(f"Project '{namespace}/{name}:{tag}' does not exist.") elif response.status_code == ResponseStatusCodes.CONFLICT: @@ -139,6 +145,9 @@ def update( json=sample_dict, ) if response.status_code == ResponseStatusCodes.ACCEPTED: + _LOGGER.info( + f"Sample '{sample_name}' updated in project '{namespace}/{name}:{tag}' successfully." + ) return None elif response.status_code == ResponseStatusCodes.NOT_EXIST: raise ResponseError( @@ -171,6 +180,9 @@ def remove(self, namespace: str, name: str, tag: str, sample_name: str): headers=self.parse_header(self.__jwt_data), ) if response.status_code == ResponseStatusCodes.ACCEPTED: + _LOGGER.info( + f"Sample '{sample_name}' removed from project '{namespace}/{name}:{tag}' successfully." + ) return None elif response.status_code == ResponseStatusCodes.NOT_EXIST: raise ResponseError( diff --git a/pephubclient/modules/view.py b/pephubclient/modules/view.py index 5633e28..f68d36c 100644 --- a/pephubclient/modules/view.py +++ b/pephubclient/modules/view.py @@ -1,5 +1,6 @@ from typing import Union import peppy +import logging from pephubclient.helpers import RequestManager from pephubclient.constants import ( @@ -10,6 +11,8 @@ from pephubclient.exceptions import ResponseError from pephubclient.models import ProjectDict +_LOGGER = logging.getLogger("pephubclient") + class PEPHubView(RequestManager): """ @@ -92,6 +95,9 @@ def create( json=sample_list, ) if response.status_code == ResponseStatusCodes.ACCEPTED: + _LOGGER.info( + f"View '{view_name}' created in project '{namespace}/{name}:{tag}' successfully." + ) return None elif response.status_code == ResponseStatusCodes.NOT_EXIST: raise ResponseError( @@ -123,6 +129,9 @@ def delete(self, namespace: str, name: str, tag: str, view_name: str) -> None: ) if response.status_code == ResponseStatusCodes.ACCEPTED: + _LOGGER.info( + f"View '{view_name}' deleted from project '{namespace}/{name}:{tag}' successfully." + ) return None elif response.status_code == ResponseStatusCodes.NOT_EXIST: raise ResponseError("View does not exists, or you are unauthorized.") @@ -163,6 +172,9 @@ def add_sample( headers=self.parse_header(self.__jwt_data), ) if response.status_code == ResponseStatusCodes.ACCEPTED: + _LOGGER.info( + f"Sample '{sample_name}' added to view '{view_name}' in project '{namespace}/{name}:{tag}' successfully." + ) return None elif response.status_code == ResponseStatusCodes.NOT_EXIST: raise ResponseError( @@ -208,6 +220,9 @@ def remove_sample( headers=self.parse_header(self.__jwt_data), ) if response.status_code == ResponseStatusCodes.ACCEPTED: + _LOGGER.info( + f"Sample '{sample_name}' removed from view '{view_name}' in project '{namespace}/{name}:{tag}' successfully." + ) return None elif response.status_code == ResponseStatusCodes.NOT_EXIST: raise ResponseError( diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt index e091589..3dd6043 100644 --- a/requirements/requirements-all.txt +++ b/requirements/requirements-all.txt @@ -1,6 +1,7 @@ typer>=0.7.0 -peppy>=0.40.0 +peppy>=0.40.1 requests>=2.28.2 pydantic>2.5.0 pandas>=2.0.0 -ubiquerg>=0.6.3 \ No newline at end of file +ubiquerg>=0.6.3 +coloredlogs>=15.0.1 \ No newline at end of file From dc03117c8483adc1d94a56d9197600ecadaa3aa1 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 12 Feb 2024 20:37:34 +0100 Subject: [PATCH 17/19] fixed tests --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9943860..cdf8165 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ def read_reqs(reqs_name): scripts=None, include_package_data=True, test_suite="tests", - tests_require=read_reqs("dev"), + tests_require=read_reqs("test"), setup_requires=( ["pytest-runner"] if {"test", "pytest", "ptr"} & set(sys.argv) else [] ), From bf21b788f7655d6a11aa4e344d0bda21bc4c1069 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 12 Feb 2024 22:11:59 +0100 Subject: [PATCH 18/19] fixed pr comments --- pephubclient/helpers.py | 7 ++----- pephubclient/modules/sample.py | 4 +++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pephubclient/helpers.py b/pephubclient/helpers.py index a9f764b..85979bf 100644 --- a/pephubclient/helpers.py +++ b/pephubclient/helpers.py @@ -16,6 +16,7 @@ import requests from requests.exceptions import ConnectionError +from urllib.parse import urlencode from ubiquerg import parse_registry_path from pydantic import ValidationError @@ -76,11 +77,7 @@ def parse_query_param(pep_variables: dict) -> str: :param pep_variables: dict of query parameters :return: PEPHubClient variables transformed into string in correct format. """ - parsed_variables = [] - - for variable_name, variable_value in pep_variables.items(): - parsed_variables.append(f"{variable_name}={variable_value}") - return "?" + "&".join(parsed_variables) + return "?" + urlencode(pep_variables) @staticmethod def parse_header(jwt_data: Optional[str] = None) -> dict: diff --git a/pephubclient/modules/sample.py b/pephubclient/modules/sample.py index 0194af9..c8208d1 100644 --- a/pephubclient/modules/sample.py +++ b/pephubclient/modules/sample.py @@ -49,7 +49,9 @@ def get( if response.status_code == ResponseStatusCodes.OK: return self.decode_response(response, output_json=True) if response.status_code == ResponseStatusCodes.NOT_EXIST: - raise ResponseError("Sample does not exist.") + raise ResponseError( + f"Sample does not exist. Project: '{namespace}/{name}:{tag}'. Sample_name: '{sample_name}'" + ) elif response.status_code == ResponseStatusCodes.INTERNAL_ERROR: raise ResponseError("Internal server error. Unexpected return value.") else: From c71a4af5c1c018d01ddcc07e2dd8dfa47ef0aacc Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 12 Feb 2024 22:54:15 +0100 Subject: [PATCH 19/19] updated version --- docs/changelog.md | 20 +++++++++++++++----- pephubclient/__init__.py | 2 +- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 0a0194b..c0209da 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,13 +2,23 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. -## [0.3.0] - 2024-XX-XX +## [0.4.0] - 2024-02-12 ### Added -- Added param parent dir where peps should be saved -- Added zip option to save_pep function +- a parameter that points to where peps should be saved ([#32](https://github.com/pepkit/pephubclient/issues/32)) +- pep zipping option to `save_pep` function ([#34](https://github.com/pepkit/pephubclient/issues/34)) +- API for samples ([#29](https://github.com/pepkit/pephubclient/issues/29)) +- API for projects ([#28](https://github.com/pepkit/pephubclient/issues/28)) -### Changed -- Transferred save_pep function to helpers +### Updated +- Transferred `save_pep` function to helpers + +## [0.3.0] - 2024-01-17 +### Added +- customization of the base PEPhub URL ([#22](https://github.com/pepkit/pephubclient/issues/22)) + +### Updated +- Updated PEPhub API URL +- Increased the required pydantic version to >2.5.0 ## [0.2.2] - 2024-01-17 ### Added diff --git a/pephubclient/__init__.py b/pephubclient/__init__.py index 3200620..4cdf008 100644 --- a/pephubclient/__init__.py +++ b/pephubclient/__init__.py @@ -4,7 +4,7 @@ import coloredlogs __app_name__ = "pephubclient" -__version__ = "0.3.0" +__version__ = "0.4.0" __author__ = "Oleksandr Khoroshevskyi, Rafal Stepien"