Skip to content
This repository has been archived by the owner on Nov 13, 2024. It is now read-only.

Commit

Permalink
fix lint
Browse files Browse the repository at this point in the history
  • Loading branch information
miararoy committed Oct 3, 2023
1 parent 582c713 commit 3718647
Show file tree
Hide file tree
Showing 4 changed files with 29 additions and 15 deletions.
24 changes: 15 additions & 9 deletions resin_cli/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@
from resin.knoweldge_base import KnowledgeBase
from resin.knoweldge_base.knowledge_base import INDEX_NAME_PREFIX
from resin.tokenizer import OpenAITokenizer, Tokenizer
from resin_cli.data_loader import load_dataframe_from_path, IndexNotUniqueError, DataframeValidationError
from resin_cli.data_loader import (
load_dataframe_from_path,
IndexNotUniqueError,
DataframeValidationError)

from .app import start as start_service
from .cli_spinner import Spinner
Expand All @@ -22,12 +25,15 @@
dotenv_path = os.path.join(os.path.dirname(__file__), ".env")
load_dotenv(dotenv_path)


spinner = Spinner()


class OpenAIError(Exception):
def __init__(self, message):
super().__init__(message)


def is_healthy(url: str):
try:
health_url = os.path.join(url, "health")
Expand Down Expand Up @@ -127,34 +133,34 @@ def upsert(index_name, data_path, tokenizer_model):
Tokenizer.initialize(OpenAITokenizer, tokenizer_model)
if data_path is None:
msg = "Data path is not provided,"
+" please provide it with --data-path or set it with env var"
+ " please provide it with --data-path or set it with env var"
click.echo(click.style(msg, fg="red"), err=True)
sys.exit(1)
click.echo("Resin is going to upsert data from ", nl=False)
click.echo(click.style(f'{data_path}', fg='yellow'), nl=False)
click.echo(click.style(f'{data_path}', fg='yellow'), nl=False)
click.echo(" to index: ")
click.echo(click.style(f'{INDEX_NAME_PREFIX}{index_name} \n', fg='green'))
with spinner:
kb = KnowledgeBase(index_name=index_name)
try:
data = load_dataframe_from_path(data_path)
except IndexNotUniqueError as e:
except IndexNotUniqueError:
msg = (
"Error: the id field on the data is not unique"
+ " this will cause records to override each other on upsert"
+ " please make sure the id field is unique"
+ " please make sure the id field is unique"
)
click.echo(click.style(f, fg="red"), err=True)
click.echo(click.style(msg, fg="red"), err=True)
sys.exit(1)
except DataframeValidationError as e:
except DataframeValidationError:
msg = (
"Error: one or more rows have not passed validation"
+ " data should agree with the Document Schema on models/data_models.py"
+ " please make sure the data is valid"
)
click.echo(click.style(f, fg="red"), err=True)
click.echo(click.style(msg, fg="red"), err=True)
sys.exit(1)
except Exception as e:
except Exception:
msg = (
"Error: an unexpected error has occured in loading data from files"
+ " it may be due to issue with the data format"
Expand Down
2 changes: 1 addition & 1 deletion resin_cli/data_loader/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
from .data_loader import load_dataframe_from_path
from .data_loader import load_dataframe_from_path
4 changes: 3 additions & 1 deletion resin_cli/data_loader/data_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ def __init__(self, message):
# Call the base class constructor with the parameters it needs
super().__init__(message)


class DataframeValidationError(ValueError):
def __init__(self, message):
# Call the base class constructor with the parameters it needs
super().__init__(message)


def _validate_dataframe(df: pd.DataFrame) -> bool:
if not isinstance(df, pd.DataFrame):
raise ValueError("Dataframe must be a pandas DataFrame")
Expand All @@ -27,7 +29,7 @@ def _validate_dataframe(df: pd.DataFrame) -> bool:
Document.validate(row)

# if any row fails validation, return False
except ValidationError as e:
except ValidationError:
return False
except ValueError as e:
raise e
Expand Down
14 changes: 10 additions & 4 deletions tests/unit/cli/test_data_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,10 @@ def test_except_not_dataframe():


def test_except_not_unique():
"""Test that _validate_dataframe raises a ValueError if passed a dataframe with a non-unique index."""
"""
Test that _validate_dataframe raises a
ValueError if passed a dataframe with a non-unique index.
"""
with pytest.raises(IndexNotUniqueError):
_validate_dataframe(
pd.DataFrame(
Expand All @@ -105,13 +108,16 @@ def test_except_not_unique():


def test_all_validator_cases():
"""Test that _validate_dataframe returns True for all dataframes in all_dataframes."""
"""
Test that _validate_dataframe returns
True for all dataframes in all_dataframes.
"""
for name, df in all_dataframes_as_dict_with_name.items():
print(name)
if name.startswith("bad"):
assert _validate_dataframe(df) == False
assert not _validate_dataframe(df)
elif name.startswith("good"):
assert _validate_dataframe(df) == True
assert _validate_dataframe(df)
print("ok")


Expand Down

0 comments on commit 3718647

Please sign in to comment.