reckless: reduce github API calls

Github API calls are now ratelimited to 60 per hour.  Searching through
the subdirectory contents of lightningd/plugins will hit this limit after
two searches or installs.  The simple answer is to look for the directory
rather than verifying a valid entrypoint in a suitably-named directory
prior to cloning the repository.

While reworking the procedure to populate the directory contents via the
GithHub API, I also corrected a bug that was flagging submodules as files.

Changelog-Fixed: Reckless is less frequently be impacted by Github API limits.

tools/reckless

@@ -28,6 +28,7 @@ logging.basicConfig(
 
 
 repos = ['https://github.com/lightningd/plugins']
+GH_API_CALLS = 0
 
 
 def py_entry_guesses(name) -> list:
@@ -147,13 +148,16 @@ class InstInfo:
         if self.srctype in [Source.DIRECTORY, Source.LOCAL_REPO]:
             depth = 5
         elif self.srctype == Source.GITHUB_REPO:
-            depth = 2
+            depth = 1
 
         def search_dir(self, sub: SourceDir, subdir: bool,
                        recursion: int) -> Union[SourceDir, None]:
             assert isinstance(recursion, int)
+            # carveout for archived plugins in lightningd/plugins
+            if recursion == 0 and 'archive' in sub.name.lower():
+                pass
             # If unable to search deeper, resort to matching directory name
-            if recursion < 1:
+            elif recursion < 1:
                 if sub.name.lower() == self.name.lower():
                     # Partial success (can't check for entrypoint)
                     self.name = sub.name
@@ -379,6 +383,41 @@ def populate_local_repo(path: str) -> list:
     return basedir.contents
 
 
+def source_element_from_repo_api(member: dict):
+    # FIXME: remove this assert
+    assert isinstance(member, dict)
+    # api accessed via /contents
+    if 'type' in member and 'name' in member and 'git_url' in member:
+        if member['type'] == 'dir':
+            return SourceDir(member['git_url'], srctype=Source.GITHUB_REPO,
+                             name=member['name'])
+        elif member['type'] == 'file':
+            # Likely a submodule
+            if member['size'] == 0:
+                return SourceDir(None, srctype=Source.GITHUB_REPO,
+                                 name=member['name'])
+            return SourceFile(member['name'])
+        # FIXME: Nope, this is by the other API
+        elif member['type'] == 'commit':
+            # No path is given by the api here
+            return SourceDir(None, srctype=Source.GITHUB_REPO,
+                             name=member['name'])
+    # git_url with /tree presents results a little differently
+    elif 'type' in member and 'path' in member and 'url' in member:
+        if member['type'] not in ['tree', 'blob']:
+            logging.debug(f'  skipping {member["path"]} type={member["type"]}')
+        if member['type'] == 'tree':
+            return SourceDir(member['url'], srctype=Source.GITHUB_REPO,
+                             name=member['path'])
+        elif member['type'] == 'blob':
+            # This can be a submodule
+            if member['size'] == 0:
+                return SourceDir(member['git_url'], srctype=Source.GITHUB_REPO,
+                                 name=member['name'])
+            return SourceFile(member['path'])
+    return None
+
+
 def populate_github_repo(url: str) -> list:
     # FIXME: This probably contains leftover cruft.
     repo = url.split('/')
@@ -398,12 +437,22 @@ def populate_github_repo(url: str) -> list:
     repo_name = parsed_url.path.split('/')[start + 1]
 
     # Get details from the github API.
-    api_url = f'{API_GITHUB_COM}/repos/{repo_user}/{repo_name}/contents/'
+    if API_GITHUB_COM in url:
+        api_url = url
+    else:
+        api_url = f'{API_GITHUB_COM}/repos/{repo_user}/{repo_name}/contents/'
 
     git_url = api_url
     if "api.github.com" in git_url:
         # This lets us redirect to handle blackbox testing
+        logging.debug(f'fetching from gh API: {git_url}')
         git_url = (API_GITHUB_COM + git_url.split("api.github.com")[-1])
+    # Ratelimiting occurs for non-authenticated GH API calls at 60 in 1 hour.
+    global GH_API_CALLS
+    GH_API_CALLS += 1
+    if GH_API_CALLS > 5:
+        logging.warning('excessive github API calls. exiting.')
+        sys.exit(1)
     r = urlopen(git_url, timeout=5)
     if r.status != 200:
         return False
@@ -413,14 +462,8 @@ def populate_github_repo(url: str) -> list:
         tree = json.loads(r.read().decode())
     contents = []
     for sub in tree:
-        if 'type' in sub and 'name' in sub and 'git_url' in sub:
-            if sub['type'] == 'dir':
-                new_sub = SourceDir(sub['git_url'], srctype=Source.GITHUB_REPO,
-                                    name=sub['name'])
-                contents.append(new_sub)
-            elif sub['type'] == 'file':
-                new_file = SourceFile(sub['name'])
-                contents.append(new_file)
+        if source_element_from_repo_api(sub):
+            contents.append(source_element_from_repo_api(sub))
     return contents
 
 
@@ -1356,3 +1399,5 @@ if __name__ == '__main__':
             args.func(target)
     else:
         args.func()
+    if GH_API_CALLS > 0:
+        logging.debug(f'GitHub API call total: {GH_API_CALLS}')