Merge pull request #26 from edsu/response-status

Model response status
Florents-Tselai · Nov 2, 2023 · 3acb426 · 3acb426
2 parents 43866ef + b3e75b9
commit 3acb426
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 0 deletions.
diff --git a/tests/test_warcdb.py b/tests/test_warcdb.py
@@ -89,3 +89,15 @@ def test_http_header():
  "value": "Wget/1.21.3",
  "warc_record_id": "<urn:uuid:6E9096E2-5D54-4CD6-A157-1DE4A7040DEB>",
  } in req_headers
+
+
+def test_http_header():
+ runner = CliRunner()
+ runner.invoke(
+ warcdb_cli, ["import", db_file, str(pathlib.Path("tests/google.warc"))]
+ )
+ db = sqlite_utils.Database(db_file)
+ responses = db["response"].rows
+ assert next(responses)["http_status"] == 301
+ assert next(responses)["http_status"] == 302
+ assert next(responses)["http_status"] == 200
diff --git a/warcdb/__init__.py b/warcdb/__init__.py
@@ -182,6 +182,8 @@ def __iadd__(self, r: ArcWarcRecord):
  )
 
  elif r.rec_type == "response":
+ if r.http_headers:
+ record_dict["http_status"] = r.http_headers.get_statuscode()
  self.db.table("response").insert(
  record_dict,
  pk="warc_record_id",

diff --git a/warcdb/migrations.py b/warcdb/migrations.py
@@ -119,3 +119,8 @@ def m002_headers(db):
  FROM response, JSON_EACH(response.http_headers) AS header
  """,
  )
+
+
+@migration()
+def m003_status(db):
+ db["response"].add_column("http_status", int)