DH-4747/add db connection to get golden record api (#189)

* DH-4747/add db connection to get golden record api

* DH-4747/adding paginating to find all methods

* DH-4747/update the tests

dataherald/api/__init__.py

@@ -98,7 +98,9 @@ def delete_golden_record(self, golden_record_id: str) -> dict:
         pass
 
     @abstractmethod
-    def get_golden_records(self, page: int = 1, limit: int = 10) -> List[GoldenRecord]:
+    def get_golden_records(
+        self, db_connection_id: str = None, page: int = 1, limit: int = 10
+    ) -> List[GoldenRecord]:
         pass
 
     @abstractmethod

dataherald/api/fastapi.py

@@ -350,13 +350,17 @@ def delete_golden_record(self, golden_record_id: str) -> dict:
         return {"status": status}
 
     @override
-    def get_golden_records(self, page: int = 1, limit: int = 10) -> List[GoldenRecord]:
+    def get_golden_records(
+        self, db_connection_id: str = None, page: int = 1, limit: int = 10
+    ) -> List[GoldenRecord]:
         golden_records_repository = GoldenRecordRepository(self.storage)
-        all_records = golden_records_repository.find_all()
-        # Calculate the start and end indices for pagination
-        start_idx = (page - 1) * limit
-        end_idx = start_idx + limit
-        return all_records[start_idx:end_idx]
+        if db_connection_id:
+            return golden_records_repository.find_by(
+                {"db_connection_id": db_connection_id},
+                page=page,
+                limit=limit,
+            )
+        return golden_records_repository.find_all(page=page, limit=limit)
 
     @override
     def add_instruction(self, instruction_request: InstructionRequest) -> Instruction:
@@ -378,7 +382,7 @@ def get_instructions(
                 page=page,
                 limit=limit,
             )
-        return instruction_repository.find_all()
+        return instruction_repository.find_all(page=page, limit=limit)
 
     @override
     def delete_instruction(self, instruction_id: str) -> dict:

dataherald/db/__init__.py

@@ -40,7 +40,7 @@ def find(
         pass
 
     @abstractmethod
-    def find_all(self, collection: str) -> list:
+    def find_all(self, collection: str, page: int = 0, limit: int = 0) -> list:
         pass
 
     @abstractmethod

dataherald/db/mongo.py

@@ -57,7 +57,12 @@ def find(
         return list(cursor)
 
     @override
-    def find_all(self, collection: str) -> list:
+    def find_all(self, collection: str, page: int = 0, limit: int = 0) -> list:
+        if page > 0 and limit > 0:
+            skip_count = (page - 1) * limit
+            return list(
+                self._data_store[collection].find({}).skip(skip_count).limit(limit)
+            )
         return list(self._data_store[collection].find({}))
 
     @override

dataherald/repositories/golden_records.py

@@ -35,8 +35,14 @@ def find_by_id(self, id: str) -> GoldenRecord | None:
             return None
         return GoldenRecord(**row)
 
-    def find_all(self) -> list[GoldenRecord]:
-        rows = self.storage.find_all(DB_COLLECTION)
+    def find_by(
+        self, query: dict, page: int = 1, limit: int = 10
+    ) -> list[GoldenRecord]:
+        rows = self.storage.find(DB_COLLECTION, query, page=page, limit=limit)
+        return [GoldenRecord(id=str(row["_id"]), **row) for row in rows]
+
+    def find_all(self, page: int = 0, limit: int = 0) -> list[GoldenRecord]:
+        rows = self.storage.find_all(DB_COLLECTION, page=page, limit=limit)
         return [GoldenRecord(id=str(row["_id"]), **row) for row in rows]
 
     def delete_by_id(self, id: str) -> int:

dataherald/repositories/instructions.py

@@ -37,15 +37,10 @@ def find_by_id(self, id: str) -> Instruction | None:
 
     def find_by(self, query: dict, page: int = 1, limit: int = 10) -> list[Instruction]:
         rows = self.storage.find(DB_COLLECTION, query, page=page, limit=limit)
-        result = []
-        for row in rows:
-            obj = Instruction(**row)
-            obj.id = str(row["_id"])
-            result.append(obj)
-        return result
-
-    def find_all(self) -> list[Instruction]:
-        rows = self.storage.find_all(DB_COLLECTION)
+        return [Instruction(id=str(row["_id"]), **row) for row in rows]
+
+    def find_all(self, page: int = 0, limit: int = 0) -> list[Instruction]:
+        rows = self.storage.find_all(DB_COLLECTION, page=page, limit=limit)
         return [Instruction(id=str(row["_id"]), **row) for row in rows]
 
     def delete_by_id(self, id: str) -> int:

dataherald/server/fastapi/__init__.py

@@ -257,9 +257,11 @@ def add_golden_records(
             content=golden_records_as_dicts, status_code=status.HTTP_201_CREATED
         )
 
-    def get_golden_records(self, page: int = 1, limit: int = 10) -> List[GoldenRecord]:
+    def get_golden_records(
+        self, db_connection_id: str = None, page: int = 1, limit: int = 10
+    ) -> List[GoldenRecord]:
         """Gets golden records"""
-        return self._api.get_golden_records(page, limit)
+        return self._api.get_golden_records(db_connection_id, page, limit)
 
     def add_instruction(self, instruction_request: InstructionRequest) -> Instruction:
         """Adds an instruction"""
@@ -273,7 +275,7 @@ def add_instruction(self, instruction_request: InstructionRequest) -> Instructio
         )
 
     def get_instructions(
-        self, db_connection_id: str = "", page: int = 1, limit: int = 10
+        self, db_connection_id: str = None, page: int = 1, limit: int = 10
     ) -> List[Instruction]:
         """Gets instructions"""
         return self._api.get_instructions(db_connection_id, page, limit)

dataherald/tests/db/test_db.py

@@ -70,7 +70,7 @@ def update_or_create(self, collection: str, query: dict, obj: dict) -> int:
         return self.insert_one(collection, obj)
 
     @override
-    def find_all(self, collection: str) -> list:
+    def find_all(self, collection: str, page: int = 0, limit: int = 0) -> list:
         return self.memory[collection]
 
     @override

docs/api.golden_record.rst

@@ -99,6 +99,7 @@ Request this ``GET`` endpoint::
    :header: "Name", "Type", "Description"
    :widths: 15, 10, 30
 
+   "db_connection_id", "string", "db connection id, ``Optoinal``"
    "page", "integer", "Page number, ``Optoinal``"
    "limit", "integer", "Page size, ``Optoinal``"
 
@@ -117,5 +118,5 @@ HTTP 200 code response
 .. code-block:: rst
 
    curl -X 'GET' \
-  'http://localhost/api/v1/golden-records?page=1&limit=10' \
+  'http://localhost/api/v1/golden-records?page=1&limit=10&db_connection_id=2342344' \
   -H 'accept: application/json'