From fd5759f2f92ec64df7a296860e71b99cd9eaa675 Mon Sep 17 00:00:00 2001 From: Leon Kuchenbecker Date: Tue, 19 Nov 2024 21:03:51 +0100 Subject: [PATCH] Backport/bucket name restriction (#142) * Allow colon characters in bucket names * Change bucket id validation in s3 only and add test * Version bump --------- Co-authored-by: Christoph Zwerschke --- .pyproject_generation/pyproject_custom.toml | 2 +- pyproject.toml | 2 +- src/hexkit/providers/s3/provider.py | 8 ++++++++ tests/integration/test_s3.py | 13 +++++++++++++ 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/.pyproject_generation/pyproject_custom.toml b/.pyproject_generation/pyproject_custom.toml index 7cad0e66..697c0586 100644 --- a/.pyproject_generation/pyproject_custom.toml +++ b/.pyproject_generation/pyproject_custom.toml @@ -1,6 +1,6 @@ [project] name = "hexkit" -version = "3.7.0" +version = "3.8.0" description = "A Toolkit for Building Microservices using the Hexagonal Architecture" requires-python = ">=3.9" classifiers = [ diff --git a/pyproject.toml b/pyproject.toml index 8bbb2cb6..becaa8d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ "Intended Audience :: Developers", ] name = "hexkit" -version = "3.7.0" +version = "3.8.0" description = "A Toolkit for Building Microservices using the Hexagonal Architecture" dependencies = [ "pydantic >=2, <3", diff --git a/src/hexkit/providers/s3/provider.py b/src/hexkit/providers/s3/provider.py index 393fb4e9..fbdfe897 100644 --- a/src/hexkit/providers/s3/provider.py +++ b/src/hexkit/providers/s3/provider.py @@ -22,6 +22,7 @@ # ruff: noqa: PLR0913 import asyncio +import re from functools import lru_cache from pathlib import Path from typing import Any, Optional @@ -31,6 +32,7 @@ import botocore.config import botocore.configloader import botocore.exceptions +import botocore.handlers from boto3.s3.transfer import TransferConfig from pydantic import Field, SecretStr from pydantic_settings import BaseSettings @@ -39,6 +41,10 @@ from hexkit.utils import calc_part_size __all__ = ["ObjectStorageProtocol", "PresignedPostURL"] +# Allow colon character in bucket names to accommodate Ceph multi tenancy S3 +botocore.handlers.VALID_BUCKET = re.compile( + r"^(?:[a-zA-Z0-9_]{1,191}:)?[a-z0-9\-]{3,63}$" +) class S3Config(BaseSettings): @@ -116,6 +122,8 @@ def read_aws_config_ini(aws_config_ini: Path) -> botocore.config.Config: class S3ObjectStorage(ObjectStorageProtocol): """S3-based provider implementing the ObjectStorageProtocol.""" + _re_bucket_id = botocore.handlers.VALID_BUCKET + def __init__( self, *, diff --git a/tests/integration/test_s3.py b/tests/integration/test_s3.py index ea87d9dc..e0a1b262 100644 --- a/tests/integration/test_s3.py +++ b/tests/integration/test_s3.py @@ -105,6 +105,19 @@ async def test_object_existence_checks(s3: S3Fixture, tmp_file: FileObject): # ) +async def test_bucket_name_with_tenant(s3: S3Fixture): + """Test if bucket names containing a tenant work correctly.""" + check_bucket = s3.storage.does_bucket_exist + assert not await check_bucket("non-existing-bucket") + assert not await check_bucket("tenant:non-existing-bucket") + with pytest.raises(ObjectStorageProtocol.BucketIdValidationError): + assert not await check_bucket("tenant:invalid:bucket") + with pytest.raises(ObjectStorageProtocol.BucketIdValidationError): + assert not await check_bucket("tenant-invalid:bucket-valid") + with pytest.raises(ObjectStorageProtocol.BucketIdValidationError): + assert not await check_bucket("tenant_valid:bucket_invalid") + + async def test_get_object_etag(s3: S3Fixture, tmp_file: FileObject): # noqa: F811 """Test ETag retrieval.""" await s3.populate_file_objects([tmp_file])