feat(release): downgrade and pin rio-tiler, fix ingestor role, allow …

…extra fields in collection links (#390)

### Added
None

### Changed/Updated
- [chore: Add rio-tiler version in raster API #
385](#385)

### Fixed
- [fix: update stac and raster api title and description
#379](#379)
- [fix: backend monitoring changes
#371](#371)
- [fix: ingestor role to have s3 access
#387](#387)
- [fix(ingest): requester pays config in validation
#388](#388)
- [fix: allow extra fields in collection links to support extensions
#389](#389)

.example.env

@@ -4,6 +4,9 @@ CDK_DEFAULT_REGION=[REQUIRED IF DEPLOYING TO EXISTING VPC]
 
 STAGE=[FILL ME IN]
 
+VEDA_PROJECT_NAME=
+VEDA_PROJECT_DESCRIPTION=
+
 VEDA_DB_PGSTAC_VERSION=0.6.6
 VEDA_DB_SCHEMA_VERSION=0.1.0
 VEDA_DB_SNAPSHOT_ID=[OPTIONAL BUT **REQUIRED** FOR ALL DEPLOYMENTS AFTER BASING DEPLOYMENT ON SNAPSHOT]

ingest_api/infrastructure/config.py

@@ -1,5 +1,5 @@
 from getpass import getuser
-from typing import Optional
+from typing import List, Optional
 
 import aws_cdk
 from pydantic import AnyHttpUrl, BaseSettings, Field, constr
@@ -8,6 +8,15 @@
 
 
 class IngestorConfig(BaseSettings):
+    # S3 bucket names where TiTiler could do HEAD and GET Requests
+    # specific private and public buckets MUST be added if you want to use s3:// urls
+    # You can whitelist all bucket by setting `*`.
+    # ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-arn-format.html
+    buckets: List = ["*"]
+
+    # S3 key pattern to limit the access to specific items (e.g: "my_data/*.tif")
+    key: str = "*"
+
     stage: str = Field(
         description=" ".join(
             [
@@ -45,6 +54,11 @@ class IngestorConfig(BaseSettings):
         None, description="ARN of AWS Role used to validate access to S3 data"
     )
 
+    raster_aws_request_payer: Optional[str] = Field(
+        None,
+        description="Set optional global parameter to 'requester' if the requester agrees to pay S3 transfer costs",
+    )
+
     stac_api_url: str = Field(description="URL of STAC API used to serve STAC Items")
 
     raster_api_url: str = Field(

ingest_api/infrastructure/construct.py

@@ -73,10 +73,22 @@ def __init__(
             build_api_lambda_params["data_access_role"] = iam.Role.from_role_arn(
                 self, "data-access-role", config.raster_data_access_role_arn
             )
+
+        if config.raster_aws_request_payer:
+            lambda_env["AWS_REQUEST_PAYER"] = config.raster_aws_request_payer
+
         build_api_lambda_params["env"] = lambda_env
 
         # create lambda
         self.api_lambda = self.build_api_lambda(**build_api_lambda_params)
+        self.api_lambda.add_to_role_policy(
+            iam.PolicyStatement(
+                actions=["s3:GetObject"],
+                resources=[
+                    f"arn:aws:s3:::{bucket}/{config.key}" for bucket in config.buckets
+                ],
+            )
+        )
 
         # create API
         self.api: aws_apigatewayv2_alpha.HttpApi = self.build_api(

ingest_api/runtime/src/auth.py

@@ -26,7 +26,6 @@ def validated_token(
     required_scopes: security.SecurityScopes,
 ) -> Dict:
     # Parse & validate token
-    logger.info(f"\nToken String {token_str}")
     try:
         token = jwt.decode(
             token_str,

ingest_api/runtime/src/config.py

@@ -19,6 +19,11 @@ class Settings(BaseSettings):
         description="ARN of AWS Role used to validate access to S3 data"
     )
 
+    aws_request_payer: Optional[str] = Field(
+        None,
+        description="Set optional global parameter to 'requester' if the requester agrees to pay S3 transfer costs",
+    )
+
     stac_url: AnyHttpUrl = Field(description="URL of STAC API")
 
     userpool_id: str = Field(description="The Cognito Userpool used for authentication")

ingest_api/runtime/src/schemas.py

@@ -7,7 +7,15 @@
 from urllib.parse import urlparse
 
 import src.validators as validators
-from pydantic import BaseModel, Field, Json, PositiveInt, error_wrappers, validator
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    Json,
+    PositiveInt,
+    error_wrappers,
+    validator,
+)
 from src.schema_helpers import SpatioTemporalExtent
 from stac_pydantic import Collection, Item, shared
 from stac_pydantic.links import Link
@@ -19,6 +27,10 @@
     from src import services
 
 
+class LinkWithExtraFields(Link):
+    model_config = ConfigDict(extra="allow")
+
+
 class AccessibleAsset(shared.Asset):
     @validator("href")
     def is_accessible(cls, href):
@@ -49,7 +61,7 @@ class DashboardCollection(Collection):
     is_periodic: Optional[bool] = Field(default=False, alias="dashboard:is_periodic")
     time_density: Optional[str] = Field(default=None, alias="dashboard:time_density")
     item_assets: Optional[Dict]
-    links: Optional[List[Link]]
+    links: Optional[List[LinkWithExtraFields]]
     assets: Optional[Dict]
     extent: SpatioTemporalExtent
 

ingest_api/runtime/src/validators.py

@@ -28,9 +28,16 @@ def s3_object_is_accessible(bucket: str, key: str):
     """
     Ensure we can send HEAD requests to S3 objects.
     """
+    from src.main import settings
+
     client = boto3.client("s3", **get_s3_credentials())
     try:
-        client.head_object(Bucket=bucket, Key=key)
+        if settings.aws_request_payer:
+            client.head_object(
+                Bucket=bucket, Key=key, RequestPayer=settings.aws_request_payer
+            )
+        else:
+            client.head_object(Bucket=bucket, Key=key)
     except client.exceptions.ClientError as e:
         raise ValueError(
             f"Asset not accessible: {e.__dict__['response']['Error']['Message']}"

raster_api/infrastructure/config.py

@@ -1,6 +1,7 @@
 """Settings for Raster API - any environment variables starting with
 `VEDA_RASTER_` will overwrite the values of variables in this file
 """
+
 from typing import Dict, List, Optional
 
 from pydantic import BaseSettings, Field
@@ -79,6 +80,11 @@ class vedaRasterSettings(BaseSettings):
         description="Complete url of custom host including subdomain. When provided, override host in api integration",
     )
 
+    project_name: Optional[str] = Field(
+        "VEDA (Visualization, Exploration, and Data Analysis)",
+        description="Name of the STAC Catalog",
+    )
+
     class Config:
         """model config"""
 

raster_api/infrastructure/construct.py

@@ -1,4 +1,5 @@
 """CDK Constrcut for a Lambda based TiTiler API with pgstac extension."""
+
 import os
 import typing
 from typing import Optional
@@ -58,7 +59,15 @@ def __init__(
             memory_size=veda_raster_settings.memory,
             timeout=Duration.seconds(veda_raster_settings.timeout),
             log_retention=aws_logs.RetentionDays.ONE_WEEK,
-            environment=veda_raster_settings.env or {},
+            environment={
+                **veda_raster_settings.env,
+                "VEDA_RASTER_ENABLE_MOSAIC_SEARCH": str(
+                    veda_raster_settings.raster_enable_mosaic_search
+                ),
+                "VEDA_RASTER_ROOT_PATH": veda_raster_settings.raster_root_path,
+                "VEDA_RASTER_STAGE": stage,
+                "VEDA_RASTER_PROJECT_NAME": veda_raster_settings.project_name,
+            },
             tracing=aws_lambda.Tracing.ACTIVE,
         )
 
@@ -67,21 +76,10 @@ def __init__(
             veda_raster_function, port_range=aws_ec2.Port.tcp(5432)
         )
 
-        veda_raster_function.add_environment(
-            "VEDA_RASTER_ENABLE_MOSAIC_SEARCH",
-            str(veda_raster_settings.raster_enable_mosaic_search),
-        )
-
         veda_raster_function.add_environment(
             "VEDA_RASTER_PGSTAC_SECRET_ARN", database.pgstac.secret.secret_full_arn
         )
 
-        veda_raster_function.add_environment(
-            "VEDA_RASTER_ROOT_PATH", veda_raster_settings.raster_root_path
-        )
-
-        veda_raster_function.add_environment("VEDA_RASTER_STAGE", stage)
-
         # Optional AWS S3 requester pays global setting
         if veda_raster_settings.raster_aws_request_payer:
             veda_raster_function.add_environment(

raster_api/runtime/setup.py

@@ -7,6 +7,7 @@
 
 inst_reqs = [
     "boto3",
+    "rio-tiler==6.5.0",
     "titiler.pgstac==0.8.3",
     "titiler.core>=0.15.5,<0.16",
     "titiler.mosaic>=0.15.5,<0.16",

raster_api/runtime/src/app.py

@@ -50,7 +50,7 @@ async def lifespan(app: FastAPI):
 
 
 app = FastAPI(
-    title=settings.name,
+    title=f"{settings.project_name} Raster API",
     version=veda_raster_version,
     openapi_url="/openapi.json",
     docs_url="/docs",

raster_api/runtime/src/config.py

@@ -52,9 +52,9 @@ def get_role_credentials(role_arn: str):
 class ApiSettings(BaseSettings):
     """API settings"""
 
-    name: str = "veda-raster"
+    project_name: str = "veda"
     cors_origins: str = "*"
-    cachecontrol: str = "public, max-age=3600"
+    cachecontrol: str = "max-age=30,must-revalidate,s-maxage=604800"
     debug: bool = False
     root_path: Optional[str] = None
     stage: Optional[str] = None

stac_api/infrastructure/config.py

@@ -1,4 +1,5 @@
 """Configuration options for the Lambda backed API implementing `stac-fastapi`."""
+
 from typing import Dict, Optional
 
 from pydantic import BaseSettings, Field
@@ -33,6 +34,16 @@ class vedaSTACSettings(BaseSettings):
         description="Complete url of custom host including subdomain. When provided, override host in api integration",
     )
 
+    project_name: Optional[str] = Field(
+        "VEDA (Visualization, Exploration, and Data Analysis)",
+        description="Name of the STAC Catalog",
+    )
+
+    project_description: Optional[str] = Field(
+        "VEDA (Visualization, Exploration, and Data Analysis) is NASA's open-source Earth Science platform in the cloud.",
+        description="Description of the STAC Catalog",
+    )
+
     class Config:
         """model config"""
 

stac_api/infrastructure/construct.py

@@ -1,4 +1,5 @@
 """CDK Construct for a Lambda backed API implementing stac-fastapi."""
+
 import os
 import typing
 from typing import Optional
@@ -56,9 +57,13 @@ def __init__(
             memory_size=veda_stac_settings.memory,
             timeout=Duration.seconds(veda_stac_settings.timeout),
             environment={
+                **{k.upper(): v for k, v in veda_stac_settings.env.items()},
                 "DB_MIN_CONN_SIZE": "0",
                 "DB_MAX_CONN_SIZE": "1",
-                **{k.upper(): v for k, v in veda_stac_settings.env.items()},
+                "VEDA_STAC_ROOT_PATH": veda_stac_settings.stac_root_path,
+                "VEDA_STAC_STAGE": stage,
+                "VEDA_STAC_PROJECT_NAME": veda_stac_settings.project_name,
+                "VEDA_STAC_PROJECT_DESCRIPTION": veda_stac_settings.project_description,
             },
             log_retention=aws_logs.RetentionDays.ONE_WEEK,
             tracing=aws_lambda.Tracing.ACTIVE,
@@ -80,12 +85,6 @@ def __init__(
             "VEDA_STAC_PGSTAC_SECRET_ARN", database.pgstac.secret.secret_full_arn
         )
 
-        lambda_function.add_environment(
-            "VEDA_STAC_ROOT_PATH", veda_stac_settings.stac_root_path
-        )
-
-        lambda_function.add_environment("VEDA_STAC_STAGE", stage)
-
         integration_kwargs = dict(handler=lambda_function)
         if veda_stac_settings.custom_host:
             integration_kwargs[

stac_api/runtime/src/app.py

@@ -35,13 +35,13 @@
 
 api = VedaStacApi(
     app=FastAPI(
-        title=api_settings.name,
+        title=f"{api_settings.project_name} STAC API",
         openapi_url="/openapi.json",
         docs_url="/docs",
         root_path=api_settings.root_path,
     ),
-    title=api_settings.name,
-    description=api_settings.name,
+    title=f"{api_settings.project_name} STAC API",
+    description=api_settings.project_description,
     settings=api_settings.load_postgres_settings(),
     extensions=PgStacExtensions,
     client=VedaCrudClient(post_request_model=POSTModel),

stac_api/runtime/src/config.py

@@ -50,9 +50,10 @@ def get_secret_dict(secret_name: str):
 class _ApiSettings(pydantic.BaseSettings):
     """API settings"""
 
-    name: str = "veda-stac"
+    project_name: Optional[str] = "veda"
+    project_description: Optional[str] = None
     cors_origins: str = "*"
-    cachecontrol: str = "public, max-age=3600"
+    cachecontrol: str = "max-age=30,must-revalidate,s-maxage=604800"
     debug: bool = False
     root_path: Optional[str] = None
     pgstac_secret_arn: Optional[str]