Use msgpack.Unpacker.feed() to allow fetching large data from the str…

…eam (#118)

This is to prevent timeout error when the response is large. It works
with 3+GiB msgpack data.

tdclient/job_api.py

@@ -248,9 +248,11 @@ def job_result_format_each(self, job_id, format, header=False):
             if code != 200:
                 self.raise_error("Get job result failed", res, "")
             if format == "msgpack":
-                unpacker = msgpack.Unpacker(res, raw=False)
-                for row in unpacker:
-                    yield row
+                unpacker = msgpack.Unpacker(raw=False, max_buffer_size=1000 * 1024 ** 2)
+                for chunk in res.stream(1024 ** 2):
+                    unpacker.feed(chunk)
+                    for row in unpacker:
+                        yield row
             elif format == "json":
                 for row in codecs.getreader("utf-8")(res):
                     yield json.loads(row)

tdclient/test/test_helper.py

@@ -46,7 +46,11 @@ def read(size=None):
         else:
             return b""
 
+    def stream(size=None):
+        yield read(size)
+
     response.read.side_effect = read
+    response.stream.side_effect = stream
     return response