Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix support of container URL #2405

Merged
merged 3 commits into from
Jul 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 7 additions & 8 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,11 @@ tilecloud_chain = "tilecloud_chain.scaffolds:Create"
main = "tilecloud_chain.server:main"

[tool.poetry.dependencies]
python = ">=3.9,<3.12"
python = ">=3.10,<3.12"
c2cwsgiutils = { version = "6.0.8", extras = ["standard", "broadcast", "oauth2", "debug"] }
pyramid-mako = "1.1.0"
python-dateutil = "2.9.0.post0"
tilecloud = { version = "1.11.0", extras = ["azure", "aws", "redis", "wsgi"] }
tilecloud = { version = "1.12.1", extras = ["azure", "aws", "redis", "wsgi"] }
Jinja2 = "3.1.4"
PyYAML = "6.0.1"
Shapely = "2.0.4"
Expand Down
24 changes: 23 additions & 1 deletion tilecloud_chain/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
import c2cwsgiutils.setup_process
import jsonschema_validator
import psycopg2
from azure.identity import DefaultAzureCredential
from azure.storage.blob import BlobServiceClient, ContainerClient, ContentSettings
from c2cwsgiutils import sentry
from PIL import Image
from prometheus_client import Counter, Summary
Expand Down Expand Up @@ -379,6 +381,26 @@ def filter(self, record: Any) -> bool:
return True


def get_azure_container_client(container: str) -> ContainerClient:
"""Get the Azure blog storage client."""
if "AZURE_STORAGE_CONNECTION_STRING" in os.environ and os.environ["AZURE_STORAGE_CONNECTION_STRING"]:
return BlobServiceClient.from_connection_string(
os.environ["AZURE_STORAGE_CONNECTION_STRING"]
).get_container_client(container=container)
elif "AZURE_STORAGE_BLOB_CONTAINER_URL" in os.environ:
container_client = ContainerClient.from_container_url(os.environ["AZURE_STORAGE_BLOB_CONTAINER_URL"])
if os.environ.get("AZURE_STORAGE_BLOB_VALIDATE_CONTAINER_NAME", "true").lower() == "true":
assert (
container == container_client.container_name
), f"Container name mismatch: {container} != {container_client.container_name}"
return container_client
else:
return BlobServiceClient(
account_url=os.environ["AZURE_STORAGE_ACCOUNT_URL"],
credential=DefaultAzureCredential(),
).get_container_client(container=container)


class TileGeneration:
"""Base class of all the tile generation."""

Expand Down Expand Up @@ -802,9 +824,9 @@ def get_store(
cache_azure = cast(tilecloud_chain.configuration.CacheAzureTyped, cache)
# on Azure
cache_tilestore = AzureStorageBlobTileStore(
container=cache_azure["container"],
tilelayout=layout,
cache_control=cache_azure.get("cache_control"),
client=get_azure_container_client(cache_azure["container"]),
)
elif cache["type"] == "mbtiles":
metadata = {}
Expand Down
22 changes: 2 additions & 20 deletions tilecloud_chain/controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,7 @@
import requests
import ruamel.yaml
from azure.core.exceptions import ResourceNotFoundError
from azure.identity import DefaultAzureCredential
from azure.storage.blob import BlobServiceClient, ContainerClient, ContentSettings
from azure.storage.blob import ContentSettings
from bottle import jinja2_template
from PIL import Image
from prometheus_client import Summary
Expand All @@ -32,6 +31,7 @@
TileGeneration,
add_common_options,
configuration,
get_azure_container_client,
get_queue_store,
get_tile_matrix_identifier,
)
Expand Down Expand Up @@ -104,24 +104,6 @@ def main(args: Optional[list[str]] = None, out: Optional[IO[str]] = None) -> Non
sys.exit(1)


def get_azure_container_client(container: str) -> ContainerClient:
"""Get the Azure blog storage client."""
if "AZURE_STORAGE_CONNECTION_STRING" in os.environ and os.environ["AZURE_STORAGE_CONNECTION_STRING"]:
return BlobServiceClient.from_connection_string(
os.environ["AZURE_STORAGE_CONNECTION_STRING"]
).get_container_client(container=container)
elif "AZURE_STORAGE_BLOB_CONTAINER_URL" in os.environ:
container_client = ContainerClient.from_container_url(os.environ["AZURE_STORAGE_BLOB_CONTAINER_URL"])
if os.environ.get("AZURE_STORAGE_BLOB_VALIDATE_CONTAINER_NAME", "true").lower() == "true":
assert container == container_client.container_name
return container_client
else:
return BlobServiceClient(
account_url=os.environ["AZURE_STORAGE_ACCOUNT_URL"],
credential=DefaultAzureCredential(),
).get_container_client(container=container)


def _send(
data: Union[bytes, str], path: str, mime_type: str, cache: tilecloud_chain.configuration.Cache
) -> None:
Expand Down
12 changes: 8 additions & 4 deletions tilecloud_chain/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,13 @@
import tilecloud_chain.configuration
import tilecloud_chain.security
from tilecloud import Tile, TileCoord
from tilecloud_chain import TileGeneration, configuration, controller, internal_mapcache
from tilecloud_chain.controller import get_azure_container_client
from tilecloud_chain import (
TileGeneration,
configuration,
controller,
get_azure_container_client,
internal_mapcache,
)

_LOGGER = logging.getLogger(__name__)

Expand Down Expand Up @@ -581,11 +586,10 @@ def serve(
with _GET_TILE.labels(storage=cache["type"]).time():
tile2 = store.get_one(tile)

if tile2:
if tile2 and tile2.data is not None:
if tile2.error:
return self.error(config, 500, tile2.error, **kwargs)

assert tile2.data
assert tile2.content_type
return self.response(
config,
Expand Down
Loading