From 34c4d99cf303d226f90bead6a643e25766dbc51f Mon Sep 17 00:00:00 2001 From: Victor Jouffrey Date: Tue, 7 May 2024 13:13:16 +0200 Subject: [PATCH 1/4] Add minimal type hinting for hopsworks.login and project.get_feature_store + stop execution method --- python/hopsworks/__init__.py | 4 +++- python/hopsworks/core/execution_api.py | 18 ++++++++++++++++++ python/hopsworks/execution.py | 9 +++++++++ python/hopsworks/project.py | 5 +++-- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/python/hopsworks/__init__.py b/python/hopsworks/__init__.py index 23dcea9df..190a6405d 100644 --- a/python/hopsworks/__init__.py +++ b/python/hopsworks/__init__.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from __future__ import annotations import warnings import logging @@ -25,6 +26,7 @@ from hopsworks.client.exceptions import RestAPIError, ProjectException from hopsworks import version, constants, client from hopsworks.connection import Connection +from hopsworks import project # Needs to run before import of hsml and hsfs warnings.filterwarnings(action="ignore", category=UserWarning, module=r".*psycopg2") @@ -62,7 +64,7 @@ def login( project: str = None, api_key_value: str = None, api_key_file: str = None, -): +) -> project.Project: """Connect to [Serverless Hopsworks](https://app.hopsworks.ai) by calling the `hopsworks.login()` function with no arguments. ```python diff --git a/python/hopsworks/core/execution_api.py b/python/hopsworks/core/execution_api.py index cbe4b13b4..9258f3268 100644 --- a/python/hopsworks/core/execution_api.py +++ b/python/hopsworks/core/execution_api.py @@ -76,3 +76,21 @@ def _delete(self, job_name, id): id, ] _client._send_request("DELETE", path_params) + + def _stop(self, job_name: str, id: int) -> None: + _client = client.get_instance() + path_params = [ + "project", + self._project_id, + "jobs", + job_name, + "executions", + id, + "status", + ] + _client._send_request( + "PUT", + path_params=path_params, + data={"state": "stopped"}, + headers={"Content-Type": "application/json"}, + ) diff --git a/python/hopsworks/execution.py b/python/hopsworks/execution.py index f7221023b..cb5b0446c 100644 --- a/python/hopsworks/execution.py +++ b/python/hopsworks/execution.py @@ -212,6 +212,15 @@ def delete(self): """ self._execution_api._delete(self._job.name, self.id) + def stop(self): + """Stop the execution + !!! danger "Potentially dangerous operation" + This operation stops the execution. + # Raises + `RestAPIError`. + """ + self._execution_api._stop(self.job_name, self.id) + def await_termination(self): """Wait until execution reaches terminal state # Raises diff --git a/python/hopsworks/project.py b/python/hopsworks/project.py index 1ae45fb0a..6c20b214b 100644 --- a/python/hopsworks/project.py +++ b/python/hopsworks/project.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from __future__ import annotations import humps import json @@ -28,7 +29,7 @@ environment_api, flink_cluster_api, ) - +from hsfs import feature_store class Project: def __init__( @@ -101,7 +102,7 @@ def created(self): """Timestamp when the project was created""" return self._created - def get_feature_store(self, name: str = None): + def get_feature_store(self, name: str = None) -> feature_store.FeatureStore: """Connect to Project's Feature Store. Defaulting to the project name of default feature store. To get a From bba197887f25c5dda8b7e588a5717660db626bb9 Mon Sep 17 00:00:00 2001 From: Victor Jouffrey Date: Tue, 7 May 2024 13:22:33 +0200 Subject: [PATCH 2/4] Contributing edit to switch to ruff --- CONTRIBUTING.md | 25 +++++++++++++------------ docs/CONTRIBUTING.md | 25 +++++++++++++------------ 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 58f07bfae..b97326e6f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,23 +12,24 @@ pip install -e ".[dev]" ``` -- Install [pre-commit](https://pre-commit.com/) and then activate its hooks. pre-commit is a framework for managing and maintaining multi-language pre-commit hooks. The library uses pre-commit to ensure code-style and code formatting through [black](https://github.com/psf/black) and [flake8](https://gitlab.com/pycqa/flake8). Run the following commands from the `python` directory: +- Install [pre-commit](https://pre-commit.com/) and then activate its hooks. pre-commit is a framework for managing and maintaining multi-language pre-commit hooks. The Feature Store uses pre-commit to ensure code-style and code formatting through [ruff](https://docs.astral.sh/ruff/). Run the following commands from the `python` directory: - ```bash - cd python - pip install --user pre-commit - pre-commit install - ``` + ```bash + cd python + pip install --user pre-commit + pre-commit install + ``` Afterwards, pre-commit will run whenever you commit. -- To run formatting and code-style separately, you can configure your IDE, such as VSCode, to use black and flake8, or run them via the command line: +- To run formatting and code-style separately, you can configure your IDE, such as VSCode, to use `ruff`, or run it via the command line: - ```bash - cd python - flake8 hopsworks - black hopsworks - ``` + ```bash + # linting + ruff check python --fix + # formatting + ruff format python + ``` ### Python documentation diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 58f07bfae..b97326e6f 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -12,23 +12,24 @@ pip install -e ".[dev]" ``` -- Install [pre-commit](https://pre-commit.com/) and then activate its hooks. pre-commit is a framework for managing and maintaining multi-language pre-commit hooks. The library uses pre-commit to ensure code-style and code formatting through [black](https://github.com/psf/black) and [flake8](https://gitlab.com/pycqa/flake8). Run the following commands from the `python` directory: +- Install [pre-commit](https://pre-commit.com/) and then activate its hooks. pre-commit is a framework for managing and maintaining multi-language pre-commit hooks. The Feature Store uses pre-commit to ensure code-style and code formatting through [ruff](https://docs.astral.sh/ruff/). Run the following commands from the `python` directory: - ```bash - cd python - pip install --user pre-commit - pre-commit install - ``` + ```bash + cd python + pip install --user pre-commit + pre-commit install + ``` Afterwards, pre-commit will run whenever you commit. -- To run formatting and code-style separately, you can configure your IDE, such as VSCode, to use black and flake8, or run them via the command line: +- To run formatting and code-style separately, you can configure your IDE, such as VSCode, to use `ruff`, or run it via the command line: - ```bash - cd python - flake8 hopsworks - black hopsworks - ``` + ```bash + # linting + ruff check python --fix + # formatting + ruff format python + ``` ### Python documentation From c37ab923a80d82ab98302ccbab47ebb32f7e209e Mon Sep 17 00:00:00 2001 From: Victor Jouffrey Date: Tue, 7 May 2024 13:25:05 +0200 Subject: [PATCH 3/4] sort imports --- python/hopsworks/__init__.py | 13 +++++++------ python/hopsworks/execution.py | 7 ++++--- python/hopsworks/project.py | 13 +++++++------ 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/python/hopsworks/__init__.py b/python/hopsworks/__init__.py index 190a6405d..a0c6920d8 100644 --- a/python/hopsworks/__init__.py +++ b/python/hopsworks/__init__.py @@ -15,24 +15,25 @@ # from __future__ import annotations -import warnings +import getpass import logging import os import sys -import getpass import tempfile +import warnings from pathlib import Path -from hopsworks.client.exceptions import RestAPIError, ProjectException -from hopsworks import version, constants, client +from hopsworks import client, constants, project, version +from hopsworks.client.exceptions import ProjectException, RestAPIError from hopsworks.connection import Connection -from hopsworks import project + # Needs to run before import of hsml and hsfs warnings.filterwarnings(action="ignore", category=UserWarning, module=r".*psycopg2") -import hsml # noqa: F401, E402 import hsfs # noqa: F401, E402 +import hsml # noqa: F401, E402 + __version__ = version.__version__ diff --git a/python/hopsworks/execution.py b/python/hopsworks/execution.py index cb5b0446c..a1e983e0a 100644 --- a/python/hopsworks/execution.py +++ b/python/hopsworks/execution.py @@ -14,11 +14,12 @@ # limitations under the License. # -import humps import json -from hopsworks.engine import execution_engine -from hopsworks.core import execution_api + +import humps from hopsworks import constants, util +from hopsworks.core import execution_api +from hopsworks.engine import execution_engine class Execution: diff --git a/python/hopsworks/project.py b/python/hopsworks/project.py index 6c20b214b..80cd387be 100644 --- a/python/hopsworks/project.py +++ b/python/hopsworks/project.py @@ -15,22 +15,23 @@ # from __future__ import annotations -import humps import json -from hopsworks import util, client, constants +import humps +from hopsworks import client, constants, util from hopsworks.client.external import Client from hopsworks.core import ( - job_api, - git_api, dataset_api, - kafka_api, - opensearch_api, environment_api, flink_cluster_api, + git_api, + job_api, + kafka_api, + opensearch_api, ) from hsfs import feature_store + class Project: def __init__( self, From 24ac8620805dec28ee0196f2a63b53477e96dc85 Mon Sep 17 00:00:00 2001 From: Victor Jouffrey <37411285+vatj@users.noreply.github.com> Date: Wed, 8 May 2024 09:09:35 +0200 Subject: [PATCH 4/4] Update CONTRIBUTING.md from review Co-authored-by: Robin Andersson --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b97326e6f..564734d53 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ pip install -e ".[dev]" ``` -- Install [pre-commit](https://pre-commit.com/) and then activate its hooks. pre-commit is a framework for managing and maintaining multi-language pre-commit hooks. The Feature Store uses pre-commit to ensure code-style and code formatting through [ruff](https://docs.astral.sh/ruff/). Run the following commands from the `python` directory: +- Install [pre-commit](https://pre-commit.com/) and then activate its hooks. pre-commit is a framework for managing and maintaining multi-language pre-commit hooks. The library uses pre-commit to ensure code-style and code formatting through [ruff](https://docs.astral.sh/ruff/). Run the following commands from the `python` directory: ```bash cd python