fix(ingest): requester pays config in validation (#388)

### Issue

NASA-IMPACT/veda-data#134

### What?

- Add optional requester pays configuration to ingest API to validate
accessibility of assets in buckets that require requester pays _if_ the
titiler been configured to use requester pays

### Why?

- Ingest API verifies that all hrefs in new items are accessible to the
titiler but is failing in cases where the bucket requires requester pays
configuration

### Testing?

I deployed this manually to dev and confirmed that the change resolves
the unable to access objects errors when configured to use requester
pays.

ingest_api/infrastructure/config.py

@@ -45,6 +45,11 @@ class IngestorConfig(BaseSettings):
         None, description="ARN of AWS Role used to validate access to S3 data"
     )
 
+    raster_aws_request_payer: Optional[str] = Field(
+        None,
+        description="Set optional global parameter to 'requester' if the requester agrees to pay S3 transfer costs",
+    )
+
     stac_api_url: str = Field(description="URL of STAC API used to serve STAC Items")
 
     raster_api_url: str = Field(

ingest_api/infrastructure/construct.py

@@ -73,6 +73,10 @@ def __init__(
             build_api_lambda_params["data_access_role"] = iam.Role.from_role_arn(
                 self, "data-access-role", config.raster_data_access_role_arn
             )
+
+        if config.raster_aws_request_payer:
+            lambda_env["AWS_REQUEST_PAYER"] = config.raster_aws_request_payer
+
         build_api_lambda_params["env"] = lambda_env
 
         # create lambda

ingest_api/runtime/src/auth.py

@@ -26,7 +26,6 @@ def validated_token(
     required_scopes: security.SecurityScopes,
 ) -> Dict:
     # Parse & validate token
-    logger.info(f"\nToken String {token_str}")
     try:
         token = jwt.decode(
             token_str,

ingest_api/runtime/src/config.py

@@ -19,6 +19,11 @@ class Settings(BaseSettings):
         description="ARN of AWS Role used to validate access to S3 data"
     )
 
+    aws_request_payer: Optional[str] = Field(
+        None,
+        description="Set optional global parameter to 'requester' if the requester agrees to pay S3 transfer costs",
+    )
+
     stac_url: AnyHttpUrl = Field(description="URL of STAC API")
 
     userpool_id: str = Field(description="The Cognito Userpool used for authentication")

ingest_api/runtime/src/validators.py

@@ -28,9 +28,16 @@ def s3_object_is_accessible(bucket: str, key: str):
     """
     Ensure we can send HEAD requests to S3 objects.
     """
+    from src.main import settings
+
     client = boto3.client("s3", **get_s3_credentials())
     try:
-        client.head_object(Bucket=bucket, Key=key)
+        if settings.aws_request_payer:
+            client.head_object(
+                Bucket=bucket, Key=key, RequestPayer=settings.aws_request_payer
+            )
+        else:
+            client.head_object(Bucket=bucket, Key=key)
     except client.exceptions.ClientError as e:
         raise ValueError(
             f"Asset not accessible: {e.__dict__['response']['Error']['Message']}"