Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat!: allow users to specify arbitrary retry stopping criteria" -m "… #280

Merged
merged 1 commit into from
Sep 4, 2024
+12 −4
Merged

feat!: allow users to specify arbitrary retry stopping criteria" -m "… #280

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions cimsparql/graphdb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
if TYPE_CHECKING:
from collections.abc import Callable

from tenacity.stop import stop_base

from cimsparql.sparql_result_json import SparqlResultValue


Expand Down Expand Up @@ -80,6 +82,7 @@
rest_api: RestApi = field(default=RestApi(os.getenv("SPARQL_REST_API", "RDF4J")))
ca_bundle: str | None = field(default=None)
retry_callback_factory: RetryCallbackFactory = field(default=RetryCallback)
retry_stop_criteria: stop_base = field(default=tenacity.stop_after_attempt(1))

# Parameters for rest api
# https://rdf4j.org/documentation/reference/rest-api/
Expand All @@ -88,7 +91,6 @@
limit: int | None = None
offset: int | None = None
timeout: int | None = None
num_retries: int = 0
max_delay_seconds: int = 60
validate: bool = False

Expand Down Expand Up @@ -241,7 +243,7 @@

sparql_result = None
for attempt in tenacity.Retrying(
stop=tenacity.stop_after_attempt(self.service_cfg.num_retries + 1),
stop=self.service_cfg.retry_stop_criteria,
wait=tenacity.wait_exponential(max=self.service_cfg.max_delay_seconds),
before=retry_cb.before,
after=retry_cb.after,
Expand All @@ -261,10 +263,10 @@
) -> tuple[pd.DataFrame, dict[str, SparqlResultValue]]:
df = (
pd.DataFrame(
sparql_result.results.values_as_dict(), columns=sparql_result.head.variables

Check failure on line 266 in cimsparql/graphdb.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest / 3.11 

Argument of type "list[str]" cannot be assigned to parameter "columns" of type "Axes | None" in function "__init__"
  Type "list[str]" is incompatible with type "Axes | None"
    "list[str]" is incompatible with "ExtensionArray"
    "list[str]" is incompatible with "ndarray[Unknown, Unknown]"
    "list[str]" is incompatible with "Index"
    "list[str]" is incompatible with "Series"
    "list[str]" is incompatible with protocol "SequenceNotStr[Unknown]"
      "index" is an incompatible type
        Type "(value: str, start: SupportsIndex = 0, stop: SupportsIndex = sys.maxsize, /) -> int" is incompatible with type "(value: Any, /, start: int = 0, stop: int = ...) -> int"
  ... (reportArgumentType)

Check failure on line 266 in cimsparql/graphdb.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest / 3.12 

Argument of type "list[str]" cannot be assigned to parameter "columns" of type "Axes | None" in function "__init__"
  Type "list[str]" is incompatible with type "Axes | None"
    "list[str]" is incompatible with "ExtensionArray"
    "list[str]" is incompatible with "ndarray[Unknown, Unknown]"
    "list[str]" is incompatible with "Index"
    "list[str]" is incompatible with "Series"
    "list[str]" is incompatible with protocol "SequenceNotStr[Unknown]"
      "index" is an incompatible type
        Type "(value: str, start: SupportsIndex = 0, stop: SupportsIndex = sys.maxsize, /) -> int" is incompatible with type "(value: Any, /, start: int = 0, stop: int = ...) -> int"
  ... (reportArgumentType)

Check failure on line 266 in cimsparql/graphdb.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest / 3.11 

Argument of type "list[str]" cannot be assigned to parameter "columns" of type "Axes | None" in function "__init__"
  Type "list[str]" is incompatible with type "Axes | None"
    "list[str]" is incompatible with "ExtensionArray"
    "list[str]" is incompatible with "ndarray[Unknown, Unknown]"
    "list[str]" is incompatible with "Index"
    "list[str]" is incompatible with "Series"
    "list[str]" is incompatible with protocol "SequenceNotStr[Unknown]"
      "index" is an incompatible type
        Type "(value: str, start: SupportsIndex = 0, stop: SupportsIndex = sys.maxsize, /) -> int" is incompatible with type "(value: Any, /, start: int = 0, stop: int = ...) -> int"
  ... (reportArgumentType)

Check failure on line 266 in cimsparql/graphdb.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest / 3.12 

Argument of type "list[str]" cannot be assigned to parameter "columns" of type "Axes | None" in function "__init__"
  Type "list[str]" is incompatible with type "Axes | None"
    "list[str]" is incompatible with "ExtensionArray"
    "list[str]" is incompatible with "ndarray[Unknown, Unknown]"
    "list[str]" is incompatible with "Index"
    "list[str]" is incompatible with "Series"
    "list[str]" is incompatible with protocol "SequenceNotStr[Unknown]"
      "index" is an incompatible type
        Type "(value: str, start: SupportsIndex = 0, stop: SupportsIndex = sys.maxsize, /) -> int" is incompatible with type "(value: Any, /, start: int = 0, stop: int = ...) -> int"
  ... (reportArgumentType)
)
if len(sparql_result.results.bindings)
else pd.DataFrame(columns=sparql_result.head.variables)

Check failure on line 269 in cimsparql/graphdb.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest / 3.11 

Argument of type "list[str]" cannot be assigned to parameter "columns" of type "Axes | None" in function "__init__"
  Type "list[str]" is incompatible with type "Axes | None"
    "list[str]" is incompatible with "ExtensionArray"
    "list[str]" is incompatible with "ndarray[Unknown, Unknown]"
    "list[str]" is incompatible with "Index"
    "list[str]" is incompatible with "Series"
    "list[str]" is incompatible with protocol "SequenceNotStr[Unknown]"
      "index" is an incompatible type
        Type "(value: str, start: SupportsIndex = 0, stop: SupportsIndex = sys.maxsize, /) -> int" is incompatible with type "(value: Any, /, start: int = 0, stop: int = ...) -> int"
  ... (reportArgumentType)

Check failure on line 269 in cimsparql/graphdb.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest / 3.12 

Argument of type "list[str]" cannot be assigned to parameter "columns" of type "Axes | None" in function "__init__"
  Type "list[str]" is incompatible with type "Axes | None"
    "list[str]" is incompatible with "ExtensionArray"
    "list[str]" is incompatible with "ndarray[Unknown, Unknown]"
    "list[str]" is incompatible with "Index"
    "list[str]" is incompatible with "Series"
    "list[str]" is incompatible with protocol "SequenceNotStr[Unknown]"
      "index" is an incompatible type
        Type "(value: str, start: SupportsIndex = 0, stop: SupportsIndex = sys.maxsize, /) -> int" is incompatible with type "(value: Any, /, start: int = 0, stop: int = ...) -> int"
  ... (reportArgumentType)

Check failure on line 269 in cimsparql/graphdb.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest / 3.11 

Argument of type "list[str]" cannot be assigned to parameter "columns" of type "Axes | None" in function "__init__"
  Type "list[str]" is incompatible with type "Axes | None"
    "list[str]" is incompatible with "ExtensionArray"
    "list[str]" is incompatible with "ndarray[Unknown, Unknown]"
    "list[str]" is incompatible with "Index"
    "list[str]" is incompatible with "Series"
    "list[str]" is incompatible with protocol "SequenceNotStr[Unknown]"
      "index" is an incompatible type
        Type "(value: str, start: SupportsIndex = 0, stop: SupportsIndex = sys.maxsize, /) -> int" is incompatible with type "(value: Any, /, start: int = 0, stop: int = ...) -> int"
  ... (reportArgumentType)

Check failure on line 269 in cimsparql/graphdb.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest / 3.12 

Argument of type "list[str]" cannot be assigned to parameter "columns" of type "Axes | None" in function "__init__"
  Type "list[str]" is incompatible with type "Axes | None"
    "list[str]" is incompatible with "ExtensionArray"
    "list[str]" is incompatible with "ndarray[Unknown, Unknown]"
    "list[str]" is incompatible with "Index"
    "list[str]" is incompatible with "Series"
    "list[str]" is incompatible with protocol "SequenceNotStr[Unknown]"
      "index" is an incompatible type
        Type "(value: str, start: SupportsIndex = 0, stop: SupportsIndex = sys.maxsize, /) -> int" is incompatible with type "(value: Any, /, start: int = 0, stop: int = ...) -> int"
  ... (reportArgumentType)
)
return df, data_row(sparql_result.head.variables, sparql_result.results.bindings)

Expand Down Expand Up @@ -355,7 +357,7 @@
)
response.raise_for_status()

@require_rdf4j

Check failure on line 360 in cimsparql/graphdb.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest / 3.11 

Argument of type "(self: Self@GraphDBClient, prefix: str, value: str) -> None" cannot be assigned to parameter "f" of type "(GraphDBClient, **P@require_rdf4j) -> T@require_rdf4j" in function "require_rdf4j"
  Type "(self: Self@GraphDBClient, prefix: str, value: str) -> None" is incompatible with type "(GraphDBClient, **P@require_rdf4j) -> T@require_rdf4j"
    Parameter 1: type "GraphDBClient" is incompatible with type "Self@GraphDBClient"
      Type "GraphDBClient" is incompatible with type "Self@GraphDBClient" (reportArgumentType)

Check failure on line 360 in cimsparql/graphdb.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest / 3.12 

Argument of type "(self: Self@GraphDBClient, prefix: str, value: str) -> None" cannot be assigned to parameter "f" of type "(GraphDBClient, **P@require_rdf4j) -> T@require_rdf4j" in function "require_rdf4j"
  Type "(self: Self@GraphDBClient, prefix: str, value: str) -> None" is incompatible with type "(GraphDBClient, **P@require_rdf4j) -> T@require_rdf4j"
    Parameter 1: type "GraphDBClient" is incompatible with type "Self@GraphDBClient"
      Type "GraphDBClient" is incompatible with type "Self@GraphDBClient" (reportArgumentType)

Check failure on line 360 in cimsparql/graphdb.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest / 3.11 

Argument of type "(self: Self@GraphDBClient, prefix: str, value: str) -> None" cannot be assigned to parameter "f" of type "(GraphDBClient, **P@require_rdf4j) -> T@require_rdf4j" in function "require_rdf4j"
  Type "(self: Self@GraphDBClient, prefix: str, value: str) -> None" is incompatible with type "(GraphDBClient, **P@require_rdf4j) -> T@require_rdf4j"
    Parameter 1: type "GraphDBClient" is incompatible with type "Self@GraphDBClient"
      Type "GraphDBClient" is incompatible with type "Self@GraphDBClient" (reportArgumentType)

Check failure on line 360 in cimsparql/graphdb.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest / 3.12 

Argument of type "(self: Self@GraphDBClient, prefix: str, value: str) -> None" cannot be assigned to parameter "f" of type "(GraphDBClient, **P@require_rdf4j) -> T@require_rdf4j" in function "require_rdf4j"
  Type "(self: Self@GraphDBClient, prefix: str, value: str) -> None" is incompatible with type "(GraphDBClient, **P@require_rdf4j) -> T@require_rdf4j"
    Parameter 1: type "GraphDBClient" is incompatible with type "Self@GraphDBClient"
      Type "GraphDBClient" is incompatible with type "Self@GraphDBClient" (reportArgumentType)
def set_namespace(self, prefix: str, value: str) -> None:
response = httpx.put(
self.service_cfg.url + f"/namespaces/{prefix}",
Expand All @@ -366,7 +368,7 @@
)
response.raise_for_status()

@require_rdf4j

Check failure on line 371 in cimsparql/graphdb.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest / 3.11 

Argument of type "(self: Self@GraphDBClient, prefix: str) -> str" cannot be assigned to parameter "f" of type "(GraphDBClient, **P@require_rdf4j) -> T@require_rdf4j" in function "require_rdf4j"
  Type "(self: Self@GraphDBClient, prefix: str) -> str" is incompatible with type "(GraphDBClient, **P@require_rdf4j) -> T@require_rdf4j"
    Parameter 1: type "GraphDBClient" is incompatible with type "Self@GraphDBClient"
      Type "GraphDBClient" is incompatible with type "Self@GraphDBClient" (reportArgumentType)

Check failure on line 371 in cimsparql/graphdb.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest / 3.12 

Argument of type "(self: Self@GraphDBClient, prefix: str) -> str" cannot be assigned to parameter "f" of type "(GraphDBClient, **P@require_rdf4j) -> T@require_rdf4j" in function "require_rdf4j"
  Type "(self: Self@GraphDBClient, prefix: str) -> str" is incompatible with type "(GraphDBClient, **P@require_rdf4j) -> T@require_rdf4j"
    Parameter 1: type "GraphDBClient" is incompatible with type "Self@GraphDBClient"
      Type "GraphDBClient" is incompatible with type "Self@GraphDBClient" (reportArgumentType)

Check failure on line 371 in cimsparql/graphdb.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest / 3.11 

Argument of type "(self: Self@GraphDBClient, prefix: str) -> str" cannot be assigned to parameter "f" of type "(GraphDBClient, **P@require_rdf4j) -> T@require_rdf4j" in function "require_rdf4j"
  Type "(self: Self@GraphDBClient, prefix: str) -> str" is incompatible with type "(GraphDBClient, **P@require_rdf4j) -> T@require_rdf4j"
    Parameter 1: type "GraphDBClient" is incompatible with type "Self@GraphDBClient"
      Type "GraphDBClient" is incompatible with type "Self@GraphDBClient" (reportArgumentType)

Check failure on line 371 in cimsparql/graphdb.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest / 3.12 

Argument of type "(self: Self@GraphDBClient, prefix: str) -> str" cannot be assigned to parameter "f" of type "(GraphDBClient, **P@require_rdf4j) -> T@require_rdf4j" in function "require_rdf4j"
  Type "(self: Self@GraphDBClient, prefix: str) -> str" is incompatible with type "(GraphDBClient, **P@require_rdf4j) -> T@require_rdf4j"
    Parameter 1: type "GraphDBClient" is incompatible with type "Self@GraphDBClient"
      Type "GraphDBClient" is incompatible with type "Self@GraphDBClient" (reportArgumentType)
def get_namespace(self, prefix: str) -> str:
response = httpx.get(
self.service_cfg.url + f"/namespaces/{prefix}", auth=self.service_cfg.auth, timeout=5.0
Expand Down
4 changes: 3 additions & 1 deletion tests/test_retry.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,9 @@ def test_retry_sync(
url = fail_first_ok_second_server(httpserver)

config = ServiceConfig(
server=url, rest_api=RestApi.DIRECT_SPARQL_ENDPOINT, num_retries=num_retries
server=url,
rest_api=RestApi.DIRECT_SPARQL_ENDPOINT,
retry_stop_criteria=tenacity.stop_after_attempt(num_retries + 1),
)

client = GraphDBClient(config)
Expand Down
6 changes: 5 additions & 1 deletion tests/test_retry_cb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from typing import Any

import pytest
import tenacity
from SPARQLWrapper import SPARQLWrapper

from cimsparql.graphdb import GraphDBClient, ServiceConfig
Expand All @@ -26,7 +27,10 @@ def queryAndConvert(self) -> dict[str, Any]: # noqa: N802

def test_after_callback(caplog: pytest.LogCaptureFixture):
wrapper = FailFirstSparqlWrapper("http://some-sparql-endpint")
client = GraphDBClient(service_cfg=ServiceConfig(num_retries=1), sparql_wrapper=wrapper)
client = GraphDBClient(
service_cfg=ServiceConfig(retry_stop_criteria=tenacity.stop_after_attempt(2)),
sparql_wrapper=wrapper,
)
client.exec_query("# Name: Select everything\nselect * where {?s ?p ?o}")

# Expect one message to contain be logged with the query name
Expand Down
Loading