Scrap info about recently added trove games (#79)

UncleGoogle · Jan 10, 2020 · a87d55f · a87d55f
1 parent 3cd5db5
commit a87d55f
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 5 deletions.
diff --git a/src/library.py b/src/library.py
@@ -30,7 +30,9 @@ async def __call__(self, only_cache: bool = False) -> Dict[str, HumbleGame]:
             if source == SOURCE.DRM_FREE:
                 all_games.extend(self._get_subproducts(orders))
             elif source == SOURCE.TROVE:
-                all_games.extend(self._get_trove_games(self._cache.get('troves', [])))
+                all_games.extend(self._get_trove_games(
+                    self._cache.get('troves', []) + self._cache.get('troves_recent', [])
+                ))
             elif source == SOURCE.KEYS:
                 all_games.extend(self._get_keys(orders, self._settings.show_revealed_keys))
 
@@ -74,6 +76,14 @@ async def _fetch_and_update_cache(self):
                 if updated_troves:
                     self._cache['troves'] = updated_troves
 
+            try:  # scrap for newest games as sometimes they are not in standard API yet
+                recently_added = (await self._api.get_montly_trove_data())['newlyAdded']
+            except Exception as e:
+                logging.error(e)
+            else:
+                if recently_added:
+                    self._cache['troves_recent'] = recently_added
+
         self._save_cache(self._cache)
 
     async def _fetch_orders(self, cached_gamekeys: Iterable[str]) -> Dict[str, dict]:

diff --git a/src/webservice.py b/src/webservice.py
@@ -19,7 +19,8 @@ class AuthorizedHumbleAPI:
     _ORDER_URL = "/api/v1/order/{}"
 
     TROVES_PER_CHUNK = 20
-    _TROVE_SUBSCRIBER = 'subscription/home'
+    _SUBSCRIPTION_HOME = 'subscription/home'
+    _SUBSCRIPTION_TROVE = 'subscription/trove'
     _TROVE_CHUNK_URL = 'api/v1/trove/chunk?index={}'
     _TROVE_DOWNLOAD_SIGN_URL = 'api/v1/user/download/sign'
     _TROVE_REDEEM_DOWNLOAD = 'humbler/redeemdownload'
@@ -105,15 +106,29 @@ async def had_trove_subscription(self) -> bool:
         """Based on current behavior of `humblebundle.com/subscription/home`
         that is accesable only by "current and former subscribers"
         """
-        res = await self._request('get', self._TROVE_SUBSCRIBER, allow_redirects=False)
+        res = await self._request('get', self._SUBSCRIPTION_HOME, allow_redirects=False)
         if res.status == 200:
             return True
         elif res.status == 302:
             return False
         else:
-            logging.warning(f'{self._TROVE_SUBSCRIBER}, Status code: {res.status}')
+            logging.warning(f'{self._SUBSCRIPTION_HOME}, Status code: {res.status}')
             return False
-
+
+    async def get_montly_trove_data(self) -> dict:
+        """Parses a subscription/trove page to find list of recently added games.
+        Returns json containing "newlyAdded" trove games and "standardProducts" that is 
+        the same like output from api/v1/trove/chunk/index=0
+        "standardProducts" may not contain "newlyAdded" sometimes
+        """
+        res = await self._request('get', self._SUBSCRIPTION_TROVE)
+        txt = await res.text()
+        search = '<script id="webpack-monthly-trove-data" type="application/json">'
+        json_start = txt.find(search) + len(search)
+        candidate = txt[json_start:].strip()
+        parsed, _ = json.JSONDecoder().raw_decode(candidate)
+        return parsed
+
     async def get_trove_details(self, from_chunk: int=0):
         troves: List[str] = []
         index = from_chunk