From 981555ef1ffc7e91dc94be51c18a4ae9f267abfb Mon Sep 17 00:00:00 2001 From: Anderson Banihirwe Date: Fri, 16 Aug 2024 16:44:26 -0700 Subject: [PATCH] add pagination to /files endpoint --- offsets_db_api/models.py | 5 +++ offsets_db_api/routers/files.py | 72 +++++++++++++++++++++------------ offsets_db_api/sql_helpers.py | 8 ++-- 3 files changed, 55 insertions(+), 30 deletions(-) diff --git a/offsets_db_api/models.py b/offsets_db_api/models.py index beadddc..a97c333 100644 --- a/offsets_db_api/models.py +++ b/offsets_db_api/models.py @@ -207,3 +207,8 @@ class PaginatedBinnedCreditTotals(pydantic.BaseModel): class PaginatedClips(pydantic.BaseModel): pagination: Pagination data: list[ClipwithProjects] | list[dict[str, typing.Any]] + + +class PaginatedFiles(pydantic.BaseModel): + pagination: Pagination + data: list[File] | list[dict[str, typing.Any]] diff --git a/offsets_db_api/routers/files.py b/offsets_db_api/routers/files.py index c90a76d..b6ea613 100644 --- a/offsets_db_api/routers/files.py +++ b/offsets_db_api/routers/files.py @@ -1,16 +1,17 @@ import datetime -from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, Request, status from fastapi_cache.decorator import cache from sqlmodel import Session, select from offsets_db_api.cache import CACHE_NAMESPACE from offsets_db_api.database import get_engine, get_session from offsets_db_api.log import get_logger -from offsets_db_api.models import File, FileCategory, FileStatus +from offsets_db_api.models import File, FileCategory, FileStatus, PaginatedFiles, Pagination from offsets_db_api.schemas import FileURLPayload from offsets_db_api.security import check_api_key from offsets_db_api.settings import get_settings +from offsets_db_api.sql_helpers import apply_filters, apply_sorting, handle_pagination from offsets_db_api.tasks import process_files router = APIRouter() @@ -71,42 +72,61 @@ async def get_file( ) -@router.get('/', response_model=list[File], summary='List files') +@router.get('/', response_model=PaginatedFiles, summary='List files') @cache(namespace=CACHE_NAMESPACE) async def get_files( + request: Request, category: FileCategory | None = None, status: FileStatus | None = None, recorded_at_from: datetime.datetime | None = None, recorded_at_to: datetime.datetime | None = None, - limit: int = 100, - offset: int = 0, + sort: list[str] = Query( + default=['recorded_at'], + description='List of sorting parameters in the format `field_name` or `+field_name` for ascending order or `-field_name` for descending order.', + ), + current_page: int = Query(1, description='Page number', ge=1), + per_page: int = Query(100, description='Items per page', le=200, ge=1), session: Session = Depends(get_session), authorized_user: bool = Depends(check_api_key), ): """Get files""" - logger.info( - 'Getting files with filter: category=%s, status=%s, recorded_at_from=%s, recorded_at_to=%s, limit=%d, offset=%d', - category, - status, - recorded_at_from, - recorded_at_to, - limit, - offset, - ) + logger.info(f'Getting files with filter: {request.url}') + + filters = [ + ('category', category, '==', File), + ('status', status, '==', File), + ('recorded_at', recorded_at_from, '>=', File), + ('recorded_at', recorded_at_to, '<=', File), + ] statement = select(File) - if category: - statement = statement.where(File.category == category) - if status: - statement = statement.where(File.status == status) - if recorded_at_from: - statement = statement.where(File.recorded_at >= recorded_at_from) - if recorded_at_to: - statement = statement.where(File.recorded_at <= recorded_at_to) + for attribute, values, operation, model in filters: + statement = apply_filters( + statement=statement, + model=model, + attribute=attribute, + values=values, + operation=operation, + ) - statement = statement.limit(limit).offset(offset) - files = session.exec(statement).all() + if sort: + statement = apply_sorting(statement=statement, sort=sort, model=File, primary_key='id') - logger.info('Found %d files', len(files)) + total_entries, current_page, total_pages, next_page, results = handle_pagination( + statement=statement, + primary_key=File.id, + current_page=current_page, + per_page=per_page, + request=request, + session=session, + ) - return files + return PaginatedFiles( + pagination=Pagination( + total_entries=total_entries, + current_page=current_page, + total_pages=total_pages, + next_page=next_page, + ), + data=results, + ) diff --git a/offsets_db_api/sql_helpers.py b/offsets_db_api/sql_helpers.py index 75619f2..8717df8 100644 --- a/offsets_db_api/sql_helpers.py +++ b/offsets_db_api/sql_helpers.py @@ -6,7 +6,7 @@ from sqlmodel import Session, and_, asc, desc, distinct, func, nullslast, or_, select from sqlmodel.sql.expression import Select as _Select -from offsets_db_api.models import Clip, ClipProject, Credit, Project +from offsets_db_api.models import Clip, ClipProject, Credit, File, Project from offsets_db_api.query_helpers import _generate_next_page_url from offsets_db_api.schemas import Registries @@ -15,7 +15,7 @@ def apply_sorting( *, statement: _Select[typing.Any], sort: list[str], - model, + model: type[Credit | Project | Clip | ClipProject | File], primary_key: str, ) -> _Select[typing.Any]: # Define valid column names @@ -55,7 +55,7 @@ def apply_sorting( def apply_filters( *, statement: _Select[typing.Any], - model: type[Credit | Project | Clip | ClipProject], + model: type[Credit | Project | Clip | ClipProject | File], attribute: str, values: list[str] | None | int | datetime.date | list[Registries], operation: str, @@ -130,7 +130,7 @@ def apply_filters( def handle_pagination( *, statement: _Select[typing.Any], - primary_key, + primary_key: typing.Any, current_page: int, per_page: int, request: Request,