Added generation of store manifests to python api

hera_librarian/client.py

@@ -17,6 +17,10 @@
     AdminCreateFileRequest,
     AdminCreateFileResponse,
     AdminRequestFailedResponse,
+    AdminStoreListItem,
+    AdminStoreListResponse,
+    AdminStoreManifestRequest,
+    AdminStoreManifestResponse,
 )
 from .models.errors import (
     ErrorClearRequest,
@@ -733,3 +737,54 @@ def add_file_row(
                 raise LibrarianError(f"Unknown error. {e}")
 
         return response
+
+    def get_store_list(
+        self,
+    ) -> list[AdminStoreListItem]:
+        """
+        Get the list of stores on this librarian.
+
+        Returns
+        -------
+        list[AdminStoreListResponse]
+            The list of stores.
+        """
+
+        response: AdminStoreListResponse = self.post(
+            endpoint="admin/store_list",
+            response=AdminStoreListResponse,
+        )
+
+        return response.root
+
+    def get_store_manifest(
+        self,
+        store_name: str,
+    ) -> AdminStoreManifestResponse:
+        """
+        Get the manifest of a store on this librarian.
+
+        Parameters
+        ----------
+        store_name : str
+            The name of the store to get the manifest for.
+
+        Returns
+        -------
+        AdminStoreManifestResponse
+            The manifest of the store.
+        """
+
+        try:
+            response: AdminStoreManifestResponse = self.post(
+                endpoint="admin/store_manifest",
+                request=AdminStoreManifestRequest(store_name=store_name),
+                response=AdminStoreManifestResponse,
+            )
+        except LibrarianHTTPError as e:
+            if e.status_code == 400 and "Store" in e.reason:
+                raise LibrarianError(e.reason)
+            else:
+                raise e
+
+        return response

hera_librarian/models/admin.py

@@ -4,7 +4,7 @@
 
 from datetime import datetime
 
-from pydantic import BaseModel
+from pydantic import BaseModel, RootModel
 
 
 class AdminCreateFileRequest(BaseModel):
@@ -44,5 +44,62 @@ class AdminCreateFileResponse(BaseModel):
 class AdminRequestFailedResponse(BaseModel):
     reason: str
     "The reason why the search failed."
+
     suggested_remedy: str
     "A suggested remedy for the failure."
+
+
+class AdminStoreListItem(BaseModel):
+    name: str
+    "The name of the store."
+
+    store_type: str
+    "The type of the store."
+
+    free_space: int
+    "The amount of space available on the store (in bytes)."
+
+    ingestable: bool
+    "Whether this store is ingestable or not."
+
+    available: bool
+    "Whether this store is available or not."
+
+
+AdminStoreListResponse = RootModel[list[AdminStoreListItem]]
+
+
+class ManifestEntry(BaseModel):
+    name: str
+    "The name of the file."
+    create_time: datetime
+    "The time the file was created."
+    size: int
+    "The size of the file in bytes."
+    checksum: str
+    "The checksum of the file."
+    uploader: str
+    "The uploader of the file."
+    source: str
+    "The source of the file."
+
+    instance_path: str
+    "The path to the instance on the store."
+    deletion_policy: str
+    "The deletion policy of the instance."
+    instance_create_time: datetime
+    "The time the instance was created."
+    instance_available: bool
+
+
+class AdminStoreManifestRequest(BaseModel):
+    store_name: str
+    "The name of the store to get the manifest for."
+
+
+class AdminStoreManifestResponse(BaseModel):
+    store_name: str
+    "The name of the store."
+
+    store_files: list[ManifestEntry]
+    "The files on the store."

hera_librarian/transfers/core.py

@@ -5,6 +5,9 @@
 from pydantic import BaseModel
 
 
+# Cannot abstract base class this as it is used
+# as a type hint currently. But that should probably
+# be changed (ValueError during pydanctic deserialization).
 class CoreTransferManager(BaseModel):
     def transfer(self, local_path: str, remote_path: str):
         """

librarian_server/api/admin.py

@@ -16,11 +16,16 @@
     AdminCreateFileRequest,
     AdminCreateFileResponse,
     AdminRequestFailedResponse,
+    AdminStoreListItem,
+    AdminStoreListResponse,
+    AdminStoreManifestRequest,
+    AdminStoreManifestResponse,
+    ManifestEntry,
 )
 
 from ..database import yield_session
 from ..orm import File, Instance, StoreMetadata
-from ..stores import StoreNames
+from ..stores import InvertedStoreNames, StoreNames
 from .auth import AdminUserDependency
 
 router = APIRouter(prefix="/api/v2/admin")
@@ -96,3 +101,80 @@ def add_file(
     session.commit()
 
     return AdminCreateFileResponse(success=True, file_exists=True)
+
+
+@router.post("/store_list")
+def store_list(
+    user: AdminUserDependency,
+    response: Response,
+    session: Session = Depends(yield_session),
+):
+    """
+    Returns a list of all stores in the database with some basic information
+    about them.
+    """
+
+    stores = session.query(StoreMetadata).all()
+
+    return AdminStoreListResponse(
+        [
+            AdminStoreListItem(
+                name=store.name,
+                store_type=InvertedStoreNames[store.store_type],
+                free_space=store.store_manager.free_space,
+                ingestable=store.ingestable,
+                available=store.store_manager.available,
+            )
+            for store in stores
+        ]
+    )
+
+
+@router.post(
+    "/store_manifest",
+    response_model=AdminStoreManifestResponse | AdminRequestFailedResponse,
+)
+def store_manifest(
+    request: AdminStoreManifestRequest,
+    user: AdminUserDependency,
+    response: Response,
+    session: Session = Depends(yield_session),
+):
+    """
+    Retrives the manifest of an entire store. Returns as JSON. This will
+    be a very large request and response, so use with caution.
+    """
+
+    # First, get the store.
+    store = (
+        session.query(StoreMetadata).filter_by(name=request.store_name).one_or_none()
+    )
+
+    if store is None:
+        response.status_code = status.HTTP_400_BAD_REQUEST
+        return AdminRequestFailedResponse(
+            reason=f"Store {request.store_name} does not exist.",
+            suggested_remedy="Create the store first. Maybe you need to run DB migration?",
+        )
+
+    # Get the list of instances.
+    instances = session.query(Instance).filter_by(store=store).all()
+
+    return AdminStoreManifestResponse(
+        store_name=store.name,
+        store_files=[
+            ManifestEntry(
+                name=instance.file.name,
+                create_time=instance.file.create_time,
+                size=instance.file.size,
+                checksum=instance.file.checksum,
+                uploader=instance.file.uploader,
+                source=instance.file.source,
+                instance_path=instance.path,
+                deletion_policy=instance.deletion_policy,
+                instance_create_time=instance.created_time,
+                instance_available=instance.available,
+            )
+            for instance in instances
+        ],
+    )

librarian_server/stores/__init__.py

@@ -17,3 +17,5 @@
     "core": 0,
     "local": 1,
 }
+
+InvertedStoreNames = {v: k for k, v in StoreNames.items()}

tests/background_unit_test/test_create_clone.py

@@ -2,6 +2,11 @@
 Tests the CreateClone background service.
 """
 
+from hera_librarian.models.admin import (
+    AdminStoreManifestRequest,
+    AdminStoreManifestResponse,
+)
+
 
 def test_create_local_clone_with_valid(
     test_client, test_server_with_valid_file, test_orm
@@ -35,7 +40,7 @@ def test_create_local_clone_with_valid(
 
     assert clone_task()
 
-    found_clone = False
+    clones = []
 
     with get_session() as session:
         instances = session.query(test_orm.Instance).all()
@@ -44,9 +49,21 @@ def test_create_local_clone_with_valid(
             assert instance.store.name != empty
 
             if instance.store.name == to_store:
-                found_clone = True
+                clones.append(instance)
+
+    assert len(clones) > 0
+
+    # Generate the manifest
+    response = test_client.post_with_auth(
+        "/api/v2/admin/store_manifest",
+        content=AdminStoreManifestRequest(store_name=to_store).model_dump_json(),
+    )
+
+    assert response.status_code == 200
+
+    manifest = AdminStoreManifestResponse.model_validate_json(response.content)
 
-    assert found_clone
+    assert len(manifest.store_files) == len(clones)
 
 
 def test_create_local_clone_with_invalid(

tests/integration_test/test_admin_services.py

@@ -93,3 +93,30 @@ def test_add_file(
             ),
             store_name="local_store",
         )
+
+
+def test_store_list(server, admin_client):
+    store_list = admin_client.get_store_list()
+
+
+def test_store_manifest(server, admin_client):
+    store_list = admin_client.get_store_list()
+
+    for store in store_list:
+        manifest = admin_client.get_store_manifest(store.name)
+
+        assert manifest.store_name == store.name
+
+        for entry in manifest.store_files:
+            assert entry.name is not None
+            assert entry.create_time is not None
+            assert entry.size is not None
+            assert entry.checksum is not None
+            assert entry.uploader is not None
+            assert entry.source is not None
+            assert entry.instance_path is not None
+            assert entry.deletion_policy is not None
+            assert entry.instance_create_time is not None
+            assert entry.instance_available is not None
+
+            assert entry.size == get_size_from_path(entry.instance_path)

tests/server_unit_test/test_admin.py

@@ -9,6 +9,8 @@
     AdminCreateFileRequest,
     AdminCreateFileResponse,
     AdminRequestFailedResponse,
+    AdminStoreListResponse,
+    AdminStoreManifestRequest,
 )
 from hera_librarian.utils import get_md5_from_path, get_size_from_path
 
@@ -134,3 +136,39 @@ def test_add_file_no_store_exists(test_client):
     response = AdminRequestFailedResponse.model_validate_json(response.content)
 
     assert response.reason == "Store not_a_store does not exist."
+
+
+def test_search_stores_and_manifest(test_client):
+    """
+    Tests that we can search for stores.
+    """
+
+    response = test_client.post_with_auth("/api/v2/admin/store_list", content="")
+
+    assert response.status_code == 200
+
+    response = AdminStoreListResponse.model_validate_json(response.content).root
+
+    # Now we can try the manifest!
+
+    new_response = test_client.post_with_auth(
+        "/api/v2/admin/store_manifest",
+        content=AdminStoreManifestRequest(
+            store_name=response[0].name
+        ).model_dump_json(),
+    )
+
+    assert new_response.status_code == 200
+
+    new_response = AdminStoreManifestRequest.model_validate_json(new_response.content)
+
+    assert new_response.store_name == response[0].name
+
+
+def test_search_manifest_no_store(test_client):
+    response = test_client.post_with_auth(
+        "/api/v2/admin/store_manifest",
+        content=AdminStoreManifestRequest(store_name="not_a_store").model_dump_json(),
+    )
+
+    assert response.status_code == 400

tests/server_unit_test/test_error.py

@@ -17,6 +17,8 @@ def test_error_to_db(test_server, test_orm):
     _, session_maker, _ = test_server
 
     with session_maker() as session:
+        starting_errors = session.query(test_orm.Error).count()
+
         log_to_database(
             ErrorSeverity.CRITICAL, ErrorCategory.DATA_AVAILABILITY, "test", session
         )
@@ -34,9 +36,9 @@ def test_error_to_db(test_server, test_orm):
     with session_maker() as session:
         errors = session.query(test_orm.Error).all()
 
-        assert len(errors) == 4
+        assert len(errors) == 4 + starting_errors
 
-        for error in errors:
+        for error in errors[starting_errors:]:
             assert error.message == "test"
             assert error.cleared is False
             assert error.cleared_time is None