Merge pull request #217 from tsm1th/u/tsm1th-add-limit-and-offset-fea…

…tures

add limit and offset features

docs/user/advanced/read.md

@@ -198,3 +198,42 @@ The following two pages cover interacting with the returned
 `~pynautobot.core.response.Record`{.interpreted-text role="py:class"}
 objects. The next page covers additional Update operations, which is
 followed by a discussion of other features and methods.
+
+## Using Pagination
+
+The Nautobot API supports pagination.  Pynautobot supports this by extending the `filter` and `all` methods the following arguments:
+
+-   limit
+-   offset
+
+The `offset` argument can only be used when specifying a `limit`.  However, `limit` can be used without specifying an `offset`.  You could use this to prevent timeouts that result from larger datasets. (By default, pynautobot will request the maximum limit supported by the server)  You can combine this with [threading](../../index.md#threading) to overcome most performance problems.  
+
+This example shows how you could chunk the same large dataset using limit.
+```python
+>>> devices = nautobot.dcim.devices.all()
+>>> len(devices)
+100
+>>> devices = nautobot.dcim.devices.all(limit=10) # Same result, but chunks to 10 requests
+>>> len(devices)
+100
+```
+
+This example shows how you could use offset to retrieve a single chunk of devices.
+```python
+>>> devices = nautobot.dcim.devices.all()
+>>> len(devices)
+100
+>>> devices = nautobot.dcim.devices.all(limit=10, offset=10) # Retrieve devices 11-20
+>>> len(devices)
+10
+```
+
+This example shows how you could filter and chunk at the same time.
+```python
+>>> devices = nautobot.dcim.devices.filter(location="DC")
+>>> len(devices)
+20
+>>> devices = nautobot.dcim.devices.filter(location="DC", limit=5) # 4 requests
+>>> len(devices)
+20
+```

pynautobot/core/endpoint.py

@@ -19,7 +19,7 @@
 from pynautobot.core.query import Request, RequestError
 from pynautobot.core.response import Record
 
-RESERVED_KWARGS = ("pk", "limit", "offset")
+RESERVED_KWARGS = ("pk",)
 
 
 def response_loader(req, return_obj, endpoint):
@@ -81,23 +81,28 @@ def _lookup_ret_obj(self, name, model):
             ret = Record
         return ret
 
-    def all(self, api_version=None):
+    def all(self, api_version=None, limit=None, offset=None):
         """Queries the 'ListView' of a given endpoint.
 
         Returns all objects from an endpoint.
 
         Args:
             api_version (str, optional): Override default or globally-set Nautobot REST API
                 version for this single request.
-
+            limit (int, optional): Overrides the max page size on
+                paginated returns.  This defines the number of records that will
+                be returned with each query to the Netbox server.  The queries
+                will be made as you iterate through the result set.
+            offset (int, optional): Overrides the offset on paginated returns.
         Returns:
             list: List of :py:class:`.Record` objects.
 
         Examples:
             >>> nb.dcim.devices.all()
             [test1-a3-oobsw2, test1-a3-oobsw3, test1-a3-oobsw4]
         """
-
+        if not limit and offset is not None:
+            raise ValueError("offset requires a positive limit value")
         api_version = api_version or self.api.api_version
         req = Request(
             base="{}/".format(self.url),
@@ -106,6 +111,8 @@ def all(self, api_version=None):
             threading=self.api.threading,
             max_workers=self.api.max_workers,
             api_version=api_version,
+            limit=limit,
+            offset=offset,
         )
 
         return response_loader(req.get(), self.return_obj, self)
@@ -219,7 +226,10 @@ def filter(self, *args, api_version=None, **kwargs):
             raise ValueError("filter must be passed kwargs. Perhaps use all() instead.")
         if any(i in RESERVED_KWARGS for i in kwargs):
             raise ValueError("A reserved {} kwarg was passed. Please remove it " "try again.".format(RESERVED_KWARGS))
-
+        limit = kwargs.pop("limit") if "limit" in kwargs else None
+        offset = kwargs.pop("offset") if "offset" in kwargs else None
+        if not limit and offset is not None:
+            raise ValueError("offset requires a positive limit value")
         api_version = api_version or self.api.api_version
         req = Request(
             filters=kwargs,
@@ -228,6 +238,8 @@ def filter(self, *args, api_version=None, **kwargs):
             http_session=self.api.http_session,
             threading=self.api.threading,
             api_version=api_version,
+            limit=limit,
+            offset=offset,
         )
 
         return response_loader(req.get(), self.return_obj, self)

pynautobot/core/query.py

@@ -128,6 +128,8 @@ def __init__(
         base,
         http_session,
         filters=None,
+        limit=None,
+        offset=None,
         key=None,
         token=None,
         threading=False,
@@ -154,6 +156,8 @@ def __init__(
         self.threading = threading
         self.max_workers = max_workers
         self.api_version = api_version
+        self.limit = limit
+        self.offset = offset
 
     def get_openapi(self):
         """Gets the OpenAPI Spec"""
@@ -314,21 +318,20 @@ def get(self, add_params=None):
             List[Response]: List of `Response` objects returned from the
                 endpoint.
         """
+        if not add_params and self.limit is not None:
+            add_params = {"limit": self.limit}
+            if self.limit and self.offset is not None:
+                add_params["offset"] = self.offset
 
-        def req_all():
+        def req_all(add_params):
             req = self._make_call(add_params=add_params)
             if isinstance(req, dict) and req.get("results") is not None:
                 ret = req["results"]
-                first_run = True
-                while req["next"]:
-                    # Not worrying about making sure add_params kwargs is
-                    # passed in here because results from detail routes aren't
-                    # paginated, thus far.
-                    if first_run:
+                while req["next"] and self.offset is None:
+                    if not add_params:
                         req = self._make_call(add_params={"limit": req["count"], "offset": len(req["results"])})
                     else:
                         req = self._make_call(url_override=req["next"])
-                    first_run = False
                     ret.extend(req["results"])
                 return ret
             else:
@@ -341,7 +344,7 @@ def req_all_threaded(add_params):
             req = self._make_call(add_params=add_params)
             if isinstance(req, dict) and req.get("results") is not None:
                 ret = req["results"]
-                if req.get("next"):
+                if req.get("next") and self.offset is None:
                     page_size = len(req["results"])
                     pages = calc_pages(page_size, req["count"])
                     page_offsets = [increment * page_size for increment in range(1, pages)]
@@ -358,7 +361,7 @@ def req_all_threaded(add_params):
         if self.threading:
             return req_all_threaded(add_params)
 
-        return req_all()
+        return req_all(add_params)
 
     def put(self, data: dict) -> dict:
         """Makes a PUT request to the Nautobot API.

tests/fixtures/dcim/interfaces_2.json

@@ -1,7 +1,7 @@
 {
     "count": 221,
     "next": null,
-    "previous": "http://localhost:8000/api/dcim/interfaces/?limit=50&offset=150",
+    "previous": "http://localhost:8000/api/dcim/interfaces/?limit=50&offset=0",
     "results": [
         {
             "id": 162,

tests/integration/test_endpoint.py

@@ -0,0 +1,24 @@
+class TestPagination:
+    """Verify we can limit and offset results on an endpoint."""
+
+    def test_all_content_types_with_offset(self, nb_client):
+        limit = 10
+        offset = 5
+        offset_cts = nb_client.extras.content_types.all(limit=limit, offset=offset)
+        assert len(offset_cts) == limit
+
+    def test_all_content_types_with_limit(self, nb_client):
+        content_types = nb_client.extras.content_types.all()
+        limited_cts = nb_client.extras.content_types.all(limit=10)
+        assert len(content_types) == len(limited_cts)
+
+    def test_filter_content_types_with_offset(self, nb_client):
+        limit = 10
+        offset = 5
+        offset_cts = nb_client.extras.content_types.filter(app_label="dcim", limit=limit, offset=offset)
+        assert len(offset_cts) == limit
+
+    def test_filter_content_types_with_limit(self, nb_client):
+        content_types = nb_client.extras.content_types.filter(app_label="dcim")
+        limited_cts = nb_client.extras.content_types.filter(app_label="dcim", limit=10)
+        assert len(content_types) == len(limited_cts)

tests/test_dcim.py

@@ -159,12 +159,28 @@ def test_modify(self, *_):
         ],
     )
     def test_get_all(self, mock):
-        ret = self.endpoint.all()
+        ret = self.endpoint.all(limit=50)
+        next_url = "http://localhost:8000/api/dcim/interfaces/?limit=50&offset=50"
         self.assertTrue(ret)
         self.assertIsInstance(ret, list)
         self.assertIsInstance(ret[0], self.ret)
         self.assertEqual(len(ret), 71)
-        mock.assert_called_with(self.bulk_uri, params={"limit": 221, "offset": 50}, json=None, headers=HEADERS)
+        mock.assert_called_with(next_url, params={}, json=None, headers=HEADERS)
+
+    @patch(
+        "requests.sessions.Session.get",
+        side_effect=[
+            Response(fixture="dcim/{}.json".format(name + "_1")),
+            Response(fixture="dcim/{}.json".format(name + "_2")),
+        ],
+    )
+    def test_get_chunk(self, mock):
+        ret = self.endpoint.all(limit=50, offset=0)
+        self.assertTrue(ret)
+        self.assertIsInstance(ret, list)
+        self.assertIsInstance(ret[0], self.ret)
+        self.assertEqual(len(ret), 50)
+        mock.assert_called_with(self.bulk_uri, params={"limit": 50, "offset": 0}, json=None, headers=HEADERS)
 
     @patch(
         "requests.sessions.Session.get",

tests/unit/test_endpoint.py

@@ -28,10 +28,24 @@ def test_filter_reserved_kwargs(self):
         test_obj = Endpoint(api, app, "test")
         with self.assertRaises(ValueError) as _:
             test_obj.filter(pk=1)
-        with self.assertRaises(ValueError) as _:
-            test_obj.filter(limit=1)
-        with self.assertRaises(ValueError) as _:
-            test_obj.filter(offset=1)
+
+    def test_all_none_limit_offset(self):
+        with patch("pynautobot.core.query.Request.get", return_value=Mock()) as mock:
+            api = Mock(base_url="http://localhost:8000/api")
+            app = Mock(name="test")
+            mock.return_value = [{"id": 123}, {"id": 321}]
+            test_obj = Endpoint(api, app, "test")
+            with self.assertRaises(ValueError) as _:
+                test_obj.all(limit=None, offset=1)
+
+    def test_filter_zero_limit_offset(self):
+        with patch("pynautobot.core.query.Request.get", return_value=Mock()) as mock:
+            api = Mock(base_url="http://localhost:8000/api")
+            app = Mock(name="test")
+            mock.return_value = [{"id": 123}, {"id": 321}]
+            test_obj = Endpoint(api, app, "test")
+            with self.assertRaises(ValueError) as _:
+                test_obj.filter(test="test", limit=0, offset=1)
 
     def test_choices(self):
         with patch("pynautobot.core.query.Request.options", return_value=Mock()) as mock: