basic support for events

github_to_sqlite/cli.py

@@ -364,6 +364,31 @@ def contributors(db_path, repos, auth):
     utils.ensure_db_shape(db)
 
 
+@cli.command()
+@click.argument(
+    "db_path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.argument("namespaces", type=str, nargs=-1)
+@click.option(
+    "-a",
+    "--auth",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=True),
+    default="auth.json",
+    help="Path to auth.json token file",
+)
+def events(db_path, namespaces, auth):
+    "Save events for the specified namespaces"
+    db = sqlite_utils.Database(db_path)
+    token = load_token(auth)
+    for ns in namespaces:
+        events = utils.fetch_events(ns, token)
+        utils.save_events(db, events, ns)
+        time.sleep(1)
+    utils.ensure_db_shape(db)
+
+
 @cli.command()
 @click.argument(
     "db_path",

github_to_sqlite/utils.py

@@ -305,8 +305,14 @@ def save_repo(db, repo):
         for key, value in repo.items()
         if (key == "html_url") or not key.endswith("url")
     }
-    to_save["owner"] = save_user(db, to_save["owner"])
-    to_save["license"] = save_license(db, to_save["license"])
+    if "owner" in to_save:
+        to_save["owner"] = save_user(db, to_save["owner"])
+    else:
+        to_save["owner"] = None
+    if "license" in to_save:
+        to_save["license"] = save_license(db, to_save["license"])
+    else:
+        to_save["license"] = None
     if "organization" in to_save:
         to_save["organization"] = save_user(db, to_save["organization"])
     else:
@@ -396,6 +402,13 @@ def fetch_contributors(repo, token=None):
         yield from contributors
 
 
+def fetch_events(ns, token=None):
+    headers = make_headers(token)
+    url = "https://api.github.com/{}/events".format(ns)
+    for events in paginate(url, headers):
+        yield from events
+
+
 def fetch_tags(repo, token=None):
     headers = make_headers(token)
     url = "https://api.github.com/repos/{}/tags".format(repo)
@@ -564,6 +577,35 @@ def save_contributors(db, contributors, repo_id):
     )
 
 
+def save_events(db, events, ns):
+    event_rows_to_add = []
+    for event in events:
+        user_id = save_user(db, event["actor"])
+        repo_id = save_repo(db, event["repo"])
+        created_at = event["created_at"]
+        public = bool(event["public"])
+        event_rows_to_add.append({
+            "user_id": user_id,
+            "repo_id": repo_id,
+            "created_at": created_at,
+            "public": public,
+            "type": event["type"],
+            "payload": event["payload"],
+        })
+    db["events"].insert_all(
+        event_rows_to_add,
+        pk=(
+            "repo_id",
+            "user_id",
+        ),
+        foreign_keys=[
+            ("repo_id", "repos", "id"),
+            ("user_id", "users", "id")
+        ],
+        replace=True,
+    )
+
+
 def save_tags(db, tags, repo_id):
     if not db["tags"].exists():
         db["tags"].create(