From 3b5cdb747bedcd80ce7f7b5bdf03eb8534163665 Mon Sep 17 00:00:00 2001 From: Javid Gafar-zada Date: Thu, 23 May 2024 15:36:09 +0200 Subject: [PATCH 1/2] [PT-5294] Improve stac resilience (#611) * Simplify stac client handling and drop unneeded token passing * Add resilience for stac metadata retrieval * Reduce duplication * Version bump and CHANGELOG.md update --- CHANGELOG.md | 10 ++++ poetry.lock | 19 ++++++- pyproject.toml | 3 +- tests/fixtures/fixtures_asset.py | 60 -------------------- tests/fixtures/fixtures_globals.py | 87 ++++------------------------- tests/fixtures/fixtures_storage.py | 6 +- tests/http/test_client.py | 3 - tests/test_asset.py | 88 ++++++++++++++++++++++++++---- tests/test_auth.py | 7 +-- up42/asset.py | 49 ++++++++--------- up42/auth.py | 8 +-- up42/http/client.py | 4 -- up42/stac_client.py | 29 ---------- up42/storage.py | 15 +++-- up42/utils.py | 15 ++++- 15 files changed, 171 insertions(+), 232 deletions(-) delete mode 100644 up42/stac_client.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2db24aeff..1056254e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,16 @@ You can check your current version with the following command: For more information, see [UP42 Python package description](https://pypi.org/project/up42-py/). +## 1.0.3a1 + +**May 23, 2024** +- Added tenacity as dependency. +- Added resilience on `asset::stac_info` and `asset::stac_items` +- Dropped pystac client subclassing +- Cleaned up fixtures +- Improved test coverage +- Dropped unneeded exposure of token + ## 1.0.2 diff --git a/poetry.lock b/poetry.lock index b4fed9ba3..ac63def50 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "affine" @@ -3379,6 +3379,21 @@ pure-eval = "*" [package.extras] tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] +[[package]] +name = "tenacity" +version = "8.3.0" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tenacity-8.3.0-py3-none-any.whl", hash = "sha256:3649f6443dbc0d9b01b9d8020a9c4ec7a1ff5f6f3c6c8a036ef371f573fe9185"}, + {file = "tenacity-8.3.0.tar.gz", hash = "sha256:953d4e6ad24357bceffbc9707bc74349aca9d245f68eb65419cf0c249a1949a2"}, +] + +[package.extras] +doc = ["reno", "sphinx"] +test = ["pytest", "tornado (>=4.5)", "typeguard"] + [[package]] name = "termcolor" version = "2.4.0" @@ -3708,4 +3723,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.9, <4" -content-hash = "96492a9954887a1b5b647adaa18ef4ab8314d184bde8ba246a70ef31e897c90c" +content-hash = "a29ca23d4b8076b7f2eb5aa17dbb1f692d45d3bce1042b0d42b572a62ff2c4fb" diff --git a/pyproject.toml b/pyproject.toml index a21a018da..f445c32cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "up42-py" -version = "1.0.2" +version = "1.0.3a1" description = "Python SDK for UP42, the geospatial marketplace and developer platform." authors = ["UP42 GmbH "] license = "https://github.com/up42/up42-py/blob/master/LICENSE" @@ -24,6 +24,7 @@ geojson = "3.1.0" geopandas = "^0.13.2" pystac-client = "^0.7.2" pyproj = "^3.6.1" +tenacity = "^8.3.0" [tool.poetry.dev-dependencies] diff --git a/tests/fixtures/fixtures_asset.py b/tests/fixtures/fixtures_asset.py index 3a4932bff..5fd329299 100644 --- a/tests/fixtures/fixtures_asset.py +++ b/tests/fixtures/fixtures_asset.py @@ -1,8 +1,5 @@ -import datetime import pathlib -import pystac -import pystac_client import pytest from up42 import asset @@ -16,32 +13,6 @@ def _asset_mock(auth_mock, requests_mock): url_asset_info = f"{constants.API_HOST}/v2/assets/{constants.ASSET_ID}/metadata" requests_mock.get(url=url_asset_info, json=constants.JSON_ASSET) - mock_item_collection = pystac.ItemCollection( - items=[ - pystac.Item( - id="test", - geometry=None, - properties={}, - bbox=None, - datetime=datetime.datetime.now(), - ) - ] - ) - - url_asset_stac_info = f"{constants.API_HOST}/v2/assets/stac/search" - - requests_mock.post( - url_asset_stac_info, - [ - {"json": constants.JSON_STORAGE_STAC}, - {"json": constants.JSON_STORAGE_STAC}, - {"json": mock_item_collection.to_dict()}, - ], - ) - - # asset stac item - requests_mock.get(url=constants.URL_STAC_CATALOG, json=constants.JSON_STAC_CATALOG_RESPONSE) - # asset update updated_json_asset = constants.JSON_ASSET.copy() updated_json_asset["title"] = "some_other_title" @@ -54,37 +25,6 @@ def _asset_mock(auth_mock, requests_mock): json={"url": constants.DOWNLOAD_URL}, ) - # stac_info url - mock_client = pystac_client.CollectionClient( - id="up42-storage", - description="UP42 Storage STAC API", - extra_fields={"up42-system:asset_id": constants.ASSET_ID}, - extent=pystac.Extent( - spatial=pystac.SpatialExtent( - bboxes=[ - [ - 13.3783333333333, - 52.4976111111112, - 13.3844444444445, - 52.5017222222223, - ] - ] - ), - temporal=pystac.TemporalExtent( - intervals=[ - [ - datetime.datetime(2021, 5, 31), - datetime.datetime(2021, 5, 31), - ] - ] - ), - ), - ) - requests_mock.get( - url=f"{constants.API_HOST}/v2/assets/stac/collections/{constants.STAC_COLLECTION_ID}", - json=mock_client.to_dict(), - ) - return asset.Asset(auth=auth_mock, asset_id=constants.ASSET_ID) diff --git a/tests/fixtures/fixtures_globals.py b/tests/fixtures/fixtures_globals.py index 7ac6a2ee2..ea9062bef 100644 --- a/tests/fixtures/fixtures_globals.py +++ b/tests/fixtures/fixtures_globals.py @@ -51,6 +51,9 @@ WEBHOOK_ID = "123" +URL_STAC_CATALOG = "https://api.up42.com/v2/assets/stac/" +URL_STAC_SEARCH = "https://api.up42.com/v2/assets/stac/search" + JSON_ASSET = { "accountId": "69353acb-f942-423f-8f32-11d6d67caa77", "createdAt": "2022-12-07T14:25:34.968Z", @@ -91,25 +94,7 @@ "empty": False, } -JSON_STORAGE_STAC = { - "links": [ - { - "href": "https://api.up42.com/v2/assets/stac/", - "rel": "root", - "type": "application/json", - }, - { - "href": "https://api.up42.com/v2/assets/stac/search", - "rel": "next", - "type": "application/json", - "body": { - "sortby": [{"field": "bbox", "direction": "desc"}], - "filter": {}, - "token": "next:12345", - }, - "method": "POST", - }, - ], +STAC_SEARCH_RESPONSE = { "type": "FeatureCollection", "features": [ { @@ -132,9 +117,10 @@ "type": "application/json", }, { - "href": "https://api.up42.com/v2/assets/stac/", + "href": URL_STAC_CATALOG, "rel": "root", "type": "application/json", + "title": "UP42 Storage", }, ], "stac_extensions": [ @@ -181,7 +167,7 @@ } -PYSTAC_MOCK_CLIENT = mock_pystac_client = { +STAC_CATALOG_RESPONSE = { "conformsTo": [ "https://api.stacspec.org/v1.0.0-rc.1/item-search#sort", "https://api.stacspec.org/v1.0.0-rc.1/collections", @@ -201,12 +187,12 @@ ], "links": [ { - "href": "https://api.up42.com/v2/assets/stac/", + "href": URL_STAC_CATALOG, "rel": "self", "type": "application/json", }, { - "href": "https://api.up42.com/v2/assets/stac/", + "href": URL_STAC_CATALOG, "rel": "root", "type": "application/json", }, @@ -216,7 +202,7 @@ "type": "application/json", }, { - "href": "https://api.up42.com/v2/assets/stac/search", + "href": URL_STAC_SEARCH, "rel": "search", "type": "application/json", "method": "POST", @@ -285,56 +271,3 @@ } JSON_BALANCE = {"data": {"balance": 10693}} - - -URL_STAC_CATALOG = "https://api.up42.com/v2/assets/stac" - -JSON_STAC_CATALOG_RESPONSE = { - "type": "Catalog", - "id": "up42-storage", - "stac_version": "1.0.0", - "description": "UP42 Storage STAC API", - "links": [ - { - "rel": "root", - "href": "https://api.up42.com/v2/assets/stac", - "type": "application/json", - "title": "UP42 Storage", - }, - { - "rel": "data", - "href": "https://api.up42.com/v2/assets/stac/collections", - "type": "application/json", - }, - { - "rel": "search", - "href": "https://api.up42.com/v2/assets/stac/search", - "type": "application/json", - "method": "POST", - }, - { - "rel": "self", - "href": "https://api.up42.com/v2/assets/stac", - "type": "application/json", - }, - ], - "stac_extensions": [], - "conformsTo": [ - "https://api.stacspec.org/v1.0.0-rc.1/collections", - "https://api.stacspec.org/v1.0.0-rc.1/core", - "http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/features-filter", - "http://www.opengis.net/spec/ogcapi-features-4/1.0/conf/simpletx", - "https://api.stacspec.org/v1.0.0-rc.1/item-search#filter", - "http://www.opengis.net/spec/cql2/1.0/conf/cql2-text", - "https://api.stacspec.org/v1.0.0-rc.1/item-search", - "https://api.stacspec.org/v1.0.0-rc.1/ogcapi-features/extensions/transaction", - "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core", - "http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/filter", - "http://www.opengis.net/spec/cql2/1.0/conf/basic-cql2", - "https://api.stacspec.org/v1.0.0-rc.1/ogcapi-features", - "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson", - "https://api.stacspec.org/v1.0.0-rc.1/item-search#sort", - "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30", - ], - "title": "UP42 Storage", -} diff --git a/tests/fixtures/fixtures_storage.py b/tests/fixtures/fixtures_storage.py index a4fa745ab..2d06632e8 100644 --- a/tests/fixtures/fixtures_storage.py +++ b/tests/fixtures/fixtures_storage.py @@ -8,16 +8,14 @@ @pytest.fixture() def storage_mock(auth_mock, requests_mock): # pystac client authentication - url_pystac_client = f"{constants.API_HOST}/v2/assets/stac" - requests_mock.get(url=url_pystac_client, json=constants.PYSTAC_MOCK_CLIENT) + requests_mock.get(url=constants.URL_STAC_CATALOG, json=constants.STAC_CATALOG_RESPONSE) # assets url_storage_assets = f"{constants.API_HOST}/v2/assets" requests_mock.get(url=url_storage_assets, json=constants.JSON_ASSETS) # storage stac - url_storage_stac = f"{constants.API_HOST}/v2/assets/stac/search" - requests_mock.post(url=url_storage_stac, json=constants.JSON_STORAGE_STAC) + requests_mock.post(url=constants.URL_STAC_SEARCH, json=constants.STAC_SEARCH_RESPONSE) # asset info url_asset_info = f"{constants.API_HOST}/v2/assets/{constants.ASSET_ID}/metadata" diff --git a/tests/http/test_client.py b/tests/http/test_client.py index 276eaaa26..f540ef915 100644 --- a/tests/http/test_client.py +++ b/tests/http/test_client.py @@ -27,11 +27,9 @@ def test_should_create_if_only_one_source_is_given( detected_settings: List[Optional[Dict]], ): detect_settings = mock.MagicMock(side_effect=detected_settings) - access_token = "some-token" retrieve = mock.sentinel.some_object detect_retriever = mock.MagicMock(return_value=retrieve) auth = mock.MagicMock() - auth.token.access_token = access_token create_auth = mock.MagicMock(return_value=auth) session = mock.MagicMock() create_session = mock.MagicMock(return_value=session) @@ -43,7 +41,6 @@ def test_should_create_if_only_one_source_is_given( create_auth=create_auth, create_session=create_session, ) - assert result.token == access_token assert result.auth == auth assert result.session == session detect_settings.assert_has_calls([mock.call(source) for source in sources]) diff --git a/tests/test_asset.py b/tests/test_asset.py index 3d62278ac..f6fda83d1 100644 --- a/tests/test_asset.py +++ b/tests/test_asset.py @@ -1,3 +1,4 @@ +import datetime import json import pathlib from typing import Dict, List, Optional @@ -70,12 +71,69 @@ def test_asset_info(asset_mock): assert asset_mock.info["name"] == constants.JSON_ASSET["name"] -def test_asset_stac_info(asset_mock): - results_stac_asset = asset_mock.stac_info - assert results_stac_asset - assert results_stac_asset.extra_fields["up42-system:asset_id"] == constants.ASSET_ID - pystac_items = asset_mock.stac_items - assert isinstance(pystac_items, pystac.ItemCollection) +class TestStacMetadata: + def test_should_get_stac_items_with_retries(self, auth_mock: up42_auth.Auth, requests_mock: req_mock.Mocker): + requests_mock.get(constants.URL_STAC_CATALOG, json=constants.STAC_CATALOG_RESPONSE) + requests_mock.post( + constants.URL_STAC_SEARCH, + [{"status_code": 401}, {"json": constants.STAC_SEARCH_RESPONSE}], + ) + asset_obj = asset.Asset(auth_mock, asset_info={"id": constants.ASSET_ID}) + expected = pystac.ItemCollection.from_dict(constants.STAC_SEARCH_RESPONSE) + assert asset_obj.stac_items.to_dict() == expected.to_dict() + + def test_fails_to_get_stac_items_after_retries(self, auth_mock: up42_auth.Auth, requests_mock: req_mock.Mocker): + requests_mock.get(constants.URL_STAC_CATALOG, json=constants.STAC_CATALOG_RESPONSE) + requests_mock.post(constants.URL_STAC_SEARCH, status_code=401) + asset_obj = asset.Asset(auth_mock, asset_info={"id": constants.ASSET_ID}) + with pytest.raises(ValueError): + _ = asset_obj.stac_items + + def test_should_get_stac_info_with_retries(self, auth_mock: up42_auth.Auth, requests_mock: req_mock.Mocker): + requests_mock.get(constants.URL_STAC_CATALOG, json=constants.STAC_CATALOG_RESPONSE) + requests_mock.post( + constants.URL_STAC_SEARCH, + [{"status_code": 401}, {"json": constants.STAC_SEARCH_RESPONSE}], + ) + expected = pystac.Collection( + id="up42-storage", + description="UP42 Storage STAC API", + extra_fields={"up42-system:asset_id": constants.ASSET_ID}, + extent=pystac.Extent( + spatial=pystac.SpatialExtent(bboxes=[[1.0, 2.0, 3.0, 4.0]]), + temporal=pystac.TemporalExtent(intervals=[[datetime.datetime.now(), None]]), + ), + ) + expected.add_link( + pystac.Link( + target=constants.URL_STAC_CATALOG, + rel="root", + title="UP42 Storage", + media_type=pystac.MediaType.JSON, + ) + ) + requests_mock.get( + url=f"{constants.API_HOST}/v2/assets/stac/collections/{constants.STAC_COLLECTION_ID}", + json=expected.to_dict(), + ) + asset_obj = asset.Asset(auth_mock, asset_info={"id": constants.ASSET_ID}) + assert asset_obj.stac_info.to_dict() == expected.to_dict() + + @pytest.mark.parametrize( + "response", + [ + {"status_code": 401}, + {"json": {"type": "FeatureCollection", "features": []}}, + ], + ) + def test_fails_to_get_stac_info_after_retries( + self, auth_mock: up42_auth.Auth, requests_mock: req_mock.Mocker, response: dict + ): + requests_mock.get(constants.URL_STAC_CATALOG, json=constants.STAC_CATALOG_RESPONSE) + requests_mock.post(constants.URL_STAC_SEARCH, [response]) + asset_obj = asset.Asset(auth_mock, asset_info={"id": constants.ASSET_ID}) + with pytest.raises(Exception): + _ = asset_obj.stac_info def match_request_body(data: Dict): @@ -92,13 +150,19 @@ class TestAssetUpdateMetadata: @pytest.mark.parametrize("title", [None, "new-title"]) @pytest.mark.parametrize("tags", [None, [], ["tag1", "tag2"]]) def test_should_update_metadata( - self, auth_mock: up42_auth.Auth, requests_mock: req_mock.Mocker, title: Optional[str], tags: Optional[List[str]] + self, + auth_mock: up42_auth.Auth, + requests_mock: req_mock.Mocker, + title: Optional[str], + tags: Optional[List[str]], ): asset_obj = asset.Asset(auth_mock, asset_info=self.asset_info) update_payload = {"title": title, "tags": tags} expected_info = {**self.asset_info, **update_payload} requests_mock.post( - url=self.endpoint_url, json=expected_info, additional_matcher=match_request_body(update_payload) + url=self.endpoint_url, + json=expected_info, + additional_matcher=match_request_body(update_payload), ) assert asset_obj.update_metadata(title=title, tags=tags) == expected_info @@ -108,7 +172,9 @@ def test_should_not_update_title_if_not_provided(self, auth_mock: up42_auth.Auth update_payload = {"tags": tags} expected_info = {**self.asset_info, **update_payload} requests_mock.post( - url=self.endpoint_url, json=expected_info, additional_matcher=match_request_body(update_payload) + url=self.endpoint_url, + json=expected_info, + additional_matcher=match_request_body(update_payload), ) assert asset_obj.update_metadata(tags=tags) == expected_info @@ -118,7 +184,9 @@ def test_should_not_update_tags_if_not_provided(self, auth_mock: up42_auth.Auth, update_payload = {"title": title} expected_info = {**self.asset_info, **update_payload} requests_mock.post( - url=self.endpoint_url, json=expected_info, additional_matcher=match_request_body(update_payload) + url=self.endpoint_url, + json=expected_info, + additional_matcher=match_request_body(update_payload), ) assert asset_obj.update_metadata(title=title) == expected_info diff --git a/tests/test_auth.py b/tests/test_auth.py index 4ac54baea..de5d9cb5d 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -61,16 +61,16 @@ def test_should_collect_credentials(self): ) +client = mock.MagicMock() session = requests.Session() session.headers = cast(MutableMapping[str, Union[str, bytes]], REQUEST_HEADERS) +client.session = session def create_auth(requests_mock: req_mock.Mocker): credential_sources = [{"some": "credentials"}] get_sources = mock.MagicMock(return_value=credential_sources) - create_client = mock.MagicMock() - create_client.return_value.token = constants.TOKEN - create_client.return_value.session = session + create_client = mock.MagicMock(return_value=client) requests_mock.get( WORKSPACE_ENDPOINT, @@ -97,7 +97,6 @@ class TestAuth: def test_should_authenticate_when_created(self, requests_mock: req_mock.Mocker): auth = create_auth(requests_mock) assert auth.workspace_id == constants.WORKSPACE_ID - assert auth.token == constants.TOKEN assert auth.session == session @pytest.mark.parametrize( diff --git a/up42/asset.py b/up42/asset.py index d02566586..174683261 100644 --- a/up42/asset.py +++ b/up42/asset.py @@ -3,9 +3,10 @@ import pystac import pystac_client +import tenacity as tnc from up42 import auth as up42_auth -from up42 import host, stac_client, utils +from up42 import host, utils logger = utils.get_logger(__name__) @@ -13,6 +14,11 @@ LIMIT = 50 NOT_PROVIDED = object() +_retry = tnc.retry( + stop=tnc.stop_after_attempt(5), + wait=tnc.wait_exponential(multiplier=1), + reraise=True, +) class Asset: @@ -52,10 +58,8 @@ def _get_info(self, asset_id: str): url = host.endpoint(f"/v2/assets/{asset_id}/metadata") return self.auth.request(request_type="GET", url=url) - @property def _stac_search(self) -> Tuple[pystac_client.Client, pystac_client.ItemSearch]: - url = host.endpoint("/v2/assets/stac") - pystac_client_aux = stac_client.PySTACAuthClient(auth=self.auth).open(url=url) + stac_client = utils.stac_client(self.auth.client.auth) stac_search_parameters = { "max_items": MAX_ITEM, "limit": LIMIT, @@ -67,29 +71,28 @@ def _stac_search(self) -> Tuple[pystac_client.Client, pystac_client.ItemSearch]: ], }, } - pystac_asset_search = pystac_client_aux.search(filter=stac_search_parameters) - return (pystac_client_aux, pystac_asset_search) + return stac_client, stac_client.search(filter=stac_search_parameters) @property - def stac_info(self) -> Optional[pystac.Collection]: + @_retry + def stac_info(self) -> Union[pystac.Collection, pystac_client.CollectionClient]: """ Gets the storage STAC information for the asset as a FeatureCollection. One asset can contain multiple STAC items (e.g. the PAN and multispectral images). """ - pystac_client_aux, pystac_asset_search = self._stac_search - resulting_item = pystac_asset_search.item_collection() - if resulting_item is None: + stac_client, stac_search = self._stac_search() + items = stac_search.item_collection() + if not items: raise ValueError(f"No STAC metadata information available for this asset {self.asset_id}") - collection_id = resulting_item[0].collection_id - return pystac_client_aux.get_collection(collection_id) + return stac_client.get_collection(items[0].collection_id) @property + @_retry def stac_items(self) -> pystac.ItemCollection: """Returns the stac items from an UP42 asset STAC representation.""" try: - _, pystac_asset_search = self._stac_search - resulting_items = pystac_asset_search.item_collection() - return resulting_items + _, stac_search = self._stac_search() + return stac_search.item_collection() except Exception as exc: raise ValueError(f"No STAC metadata information available for this asset {self.asset_id}") from exc @@ -165,17 +168,11 @@ def download( logger.info("Download directory: %s", output_directory) download_url = self._get_download_url() - if unpacking: - out_filepaths = utils.download_archive( - download_url=download_url, - output_directory=output_directory, - ) - else: - out_filepaths = utils.download_file( - download_url=download_url, - output_directory=output_directory, - ) - + download = utils.download_archive if unpacking else utils.download_file + out_filepaths = download( + download_url=download_url, + output_directory=output_directory, + ) self.results = out_filepaths return out_filepaths diff --git a/up42/auth.py b/up42/auth.py index afa1fe28b..10b36d883 100644 --- a/up42/auth.py +++ b/up42/auth.py @@ -53,17 +53,13 @@ def __init__( """ self.workspace_id: Optional[str] = None credential_sources = get_credential_sources(cfg_file, username, password) - self._client = create_client(credential_sources, host.endpoint("/oauth/token")) + self.client = create_client(credential_sources, host.endpoint("/oauth/token")) self._get_workspace() logger.info("Authentication with UP42 successful!") - @property - def token(self) -> str: - return self._client.token - @property def session(self) -> requests.Session: - return self._client.session + return self.client.session def _get_workspace(self) -> None: """Get user id belonging to authenticated account.""" diff --git a/up42/http/client.py b/up42/http/client.py index 5b9fb13fe..3d6162e8b 100644 --- a/up42/http/client.py +++ b/up42/http/client.py @@ -14,10 +14,6 @@ def __init__(self, auth: oauth.Up42Auth, create_session: SessionFactory): self.auth = auth self.session = create_session(auth) - @property - def token(self) -> str: - return self.auth.token.access_token - def _merge( left: Optional[config.CredentialsSettings], diff --git a/up42/stac_client.py b/up42/stac_client.py deleted file mode 100644 index 56d681bcc..000000000 --- a/up42/stac_client.py +++ /dev/null @@ -1,29 +0,0 @@ -from pystac_client import Client - -from up42.auth import Auth - - -class PySTACAuthClient(Client): - """Pystac Client authenticated to access stac catalog.""" - - def __init__( - self, - *args, - id="id", - description="description", - auth: Auth = None, - **kwargs, - ): # pylint: disable=redefined-builtin - super().__init__(id=id, description=description, *args, **kwargs) # type: ignore - self.auth = auth - - def _auth_modifier(self, request): - """Callable for the pystac client request_modifier to authenticate catalog calls.""" - request.headers["Authorization"] = f"Bearer {self.auth.token}" - - def open(self, *args, **kwargs) -> Client: # type: ignore # pylint: disable=arguments-differ - return super().open( # type: ignore - request_modifier=self._auth_modifier, - *args, - **kwargs, - ) diff --git a/up42/storage.py b/up42/storage.py index 03c95717b..6e98f3a1f 100644 --- a/up42/storage.py +++ b/up42/storage.py @@ -10,7 +10,7 @@ from up42 import asset, asset_searcher from up42 import auth as up42_auth -from up42 import host, order, stac_client, utils +from up42 import host, order, utils logger = utils.get_logger(__name__) @@ -48,9 +48,7 @@ def __repr__(self): @property def pystac_client(self): - url = host.endpoint("/v2/assets/stac") - pystac_client_auth = stac_client.PySTACAuthClient(auth=self.auth).open(url=url) - return pystac_client_auth + return utils.stac_client(self.auth.client.auth) def get_assets( self, @@ -59,7 +57,14 @@ def get_assets( acquired_after: Optional[Union[str, datetime.datetime]] = None, acquired_before: Optional[Union[str, datetime.datetime]] = None, geometry: Optional[ - Union[dict, geojson.Feature, geojson.FeatureCollection, list, geopandas.GeoDataFrame, shp_geometry.Polygon] + Union[ + dict, + geojson.Feature, + geojson.FeatureCollection, + list, + geopandas.GeoDataFrame, + shp_geometry.Polygon, + ] ] = None, workspace_id: Optional[str] = None, collection_names: Optional[List[str]] = None, diff --git a/up42/utils.py b/up42/utils.py index bdb283025..1824b835b 100644 --- a/up42/utils.py +++ b/up42/utils.py @@ -9,16 +9,19 @@ import tempfile import warnings import zipfile -from typing import List, Optional, Union, cast +from typing import Callable, List, Optional, Union, cast from urllib import parse import geojson # type: ignore import geopandas # type: ignore +import pystac_client import requests import shapely # type: ignore import tqdm from shapely import geometry # type: ignore +from up42 import host + TIMEOUT = 120 # seconds CHUNK_SIZE = 1024 @@ -411,3 +414,13 @@ def read_json(path_or_dict: Union[dict, str, pathlib.Path, None]) -> Optional[di except FileNotFoundError as ex: raise ValueError(f"File {path_or_dict} does not exist!") from ex return cast(Optional[dict], path_or_dict) + + +def stac_client(auth: requests.auth.AuthBase): + # pystac client accepts both returning and non-returning request modifiers + # requests.auth.AuthBase is a returning request modifier interface + request_modifier = cast(Callable[[requests.Request], Optional[requests.Request]], auth) + return pystac_client.Client.open( + url=host.endpoint("/v2/assets/stac/"), + request_modifier=request_modifier, + ) From 6832d14f26ac0a12e90dcaeb41f3f4bc7acf58e4 Mon Sep 17 00:00:00 2001 From: Andres Hernandez Date: Thu, 23 May 2024 15:50:16 +0200 Subject: [PATCH 2/2] bump to version 1.0.3 (#612) --- CHANGELOG.md | 10 ++++++++++ pyproject.toml | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1056254e7..e12f5cc09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,16 @@ You can check your current version with the following command: For more information, see [UP42 Python package description](https://pypi.org/project/up42-py/). +## 1.0.3 + +**May 23, 2024** +- Added tenacity as dependency. +- Added resilience on `asset::stac_info` and `asset::stac_items` +- Dropped pystac client subclassing +- Cleaned up fixtures +- Improved test coverage +- Dropped unneeded exposure of token + ## 1.0.3a1 **May 23, 2024** diff --git a/pyproject.toml b/pyproject.toml index f445c32cb..04e9d0565 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "up42-py" -version = "1.0.3a1" +version = "1.0.3" description = "Python SDK for UP42, the geospatial marketplace and developer platform." authors = ["UP42 GmbH "] license = "https://github.com/up42/up42-py/blob/master/LICENSE"