add pagination to /files endpoint

offsets_db_api/models.py

@@ -207,3 +207,8 @@ class PaginatedBinnedCreditTotals(pydantic.BaseModel):
 class PaginatedClips(pydantic.BaseModel):
     pagination: Pagination
     data: list[ClipwithProjects] | list[dict[str, typing.Any]]
+
+
+class PaginatedFiles(pydantic.BaseModel):
+    pagination: Pagination
+    data: list[File] | list[dict[str, typing.Any]]

offsets_db_api/routers/files.py

@@ -1,16 +1,17 @@
 import datetime
 
-from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status
+from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, Request, status
 from fastapi_cache.decorator import cache
 from sqlmodel import Session, select
 
 from offsets_db_api.cache import CACHE_NAMESPACE
 from offsets_db_api.database import get_engine, get_session
 from offsets_db_api.log import get_logger
-from offsets_db_api.models import File, FileCategory, FileStatus
+from offsets_db_api.models import File, FileCategory, FileStatus, PaginatedFiles, Pagination
 from offsets_db_api.schemas import FileURLPayload
 from offsets_db_api.security import check_api_key
 from offsets_db_api.settings import get_settings
+from offsets_db_api.sql_helpers import apply_filters, apply_sorting, handle_pagination
 from offsets_db_api.tasks import process_files
 
 router = APIRouter()
@@ -71,42 +72,61 @@ async def get_file(
         )
 
 
-@router.get('/', response_model=list[File], summary='List files')
+@router.get('/', response_model=PaginatedFiles, summary='List files')
 @cache(namespace=CACHE_NAMESPACE)
 async def get_files(
+    request: Request,
     category: FileCategory | None = None,
     status: FileStatus | None = None,
     recorded_at_from: datetime.datetime | None = None,
     recorded_at_to: datetime.datetime | None = None,
-    limit: int = 100,
-    offset: int = 0,
+    sort: list[str] = Query(
+        default=['recorded_at'],
+        description='List of sorting parameters in the format `field_name` or `+field_name` for ascending order or `-field_name` for descending order.',
+    ),
+    current_page: int = Query(1, description='Page number', ge=1),
+    per_page: int = Query(100, description='Items per page', le=200, ge=1),
     session: Session = Depends(get_session),
     authorized_user: bool = Depends(check_api_key),
 ):
     """Get files"""
-    logger.info(
-        'Getting files with filter: category=%s, status=%s, recorded_at_from=%s, recorded_at_to=%s, limit=%d, offset=%d',
-        category,
-        status,
-        recorded_at_from,
-        recorded_at_to,
-        limit,
-        offset,
-    )
+    logger.info(f'Getting files with filter: {request.url}')
+
+    filters = [
+        ('category', category, '==', File),
+        ('status', status, '==', File),
+        ('recorded_at', recorded_at_from, '>=', File),
+        ('recorded_at', recorded_at_to, '<=', File),
+    ]
 
     statement = select(File)
-    if category:
-        statement = statement.where(File.category == category)
-    if status:
-        statement = statement.where(File.status == status)
-    if recorded_at_from:
-        statement = statement.where(File.recorded_at >= recorded_at_from)
-    if recorded_at_to:
-        statement = statement.where(File.recorded_at <= recorded_at_to)
+    for attribute, values, operation, model in filters:
+        statement = apply_filters(
+            statement=statement,
+            model=model,
+            attribute=attribute,
+            values=values,
+            operation=operation,
+        )
 
-    statement = statement.limit(limit).offset(offset)
-    files = session.exec(statement).all()
+    if sort:
+        statement = apply_sorting(statement=statement, sort=sort, model=File, primary_key='id')
 
-    logger.info('Found %d files', len(files))
+    total_entries, current_page, total_pages, next_page, results = handle_pagination(
+        statement=statement,
+        primary_key=File.id,
+        current_page=current_page,
+        per_page=per_page,
+        request=request,
+        session=session,
+    )
 
-    return files
+    return PaginatedFiles(
+        pagination=Pagination(
+            total_entries=total_entries,
+            current_page=current_page,
+            total_pages=total_pages,
+            next_page=next_page,
+        ),
+        data=results,
+    )

offsets_db_api/sql_helpers.py

@@ -6,7 +6,7 @@
 from sqlmodel import Session, and_, asc, desc, distinct, func, nullslast, or_, select
 from sqlmodel.sql.expression import Select as _Select
 
-from offsets_db_api.models import Clip, ClipProject, Credit, Project
+from offsets_db_api.models import Clip, ClipProject, Credit, File, Project
 from offsets_db_api.query_helpers import _generate_next_page_url
 from offsets_db_api.schemas import Registries
 
@@ -15,7 +15,7 @@ def apply_sorting(
     *,
     statement: _Select[typing.Any],
     sort: list[str],
-    model,
+    model: type[Credit | Project | Clip | ClipProject | File],
     primary_key: str,
 ) -> _Select[typing.Any]:
     # Define valid column names
@@ -55,7 +55,7 @@ def apply_sorting(
 def apply_filters(
     *,
     statement: _Select[typing.Any],
-    model: type[Credit | Project | Clip | ClipProject],
+    model: type[Credit | Project | Clip | ClipProject | File],
     attribute: str,
     values: list[str] | None | int | datetime.date | list[Registries],
     operation: str,
@@ -130,7 +130,7 @@ def apply_filters(
 def handle_pagination(
     *,
     statement: _Select[typing.Any],
-    primary_key,
+    primary_key: typing.Any,
     current_page: int,
     per_page: int,
     request: Request,