Fixing Twitter chaotic responses

medialab · Sep 10, 2021 · 08e1ba6 · 08e1ba6
1 parent 29bbad6
commit 08e1ba6
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/minet/twitter/api_scraper.py b/minet/twitter/api_scraper.py
@@ -381,7 +381,10 @@ def search(self, query, limit=None, before_sleep=None,
                 if limit is not None and i >= limit:
                     return
 
-            if new_cursor is None or len(tweets) == 0:
+            # NOTE: we cannot stop when no tweets are found anymore because
+            # of Twitter's public facing API's strange hiccups.
+            # Comparing cursors seems to be the only good option now...
+            if new_cursor is None or new_cursor == cursor:
                 return
 
             cursor = new_cursor