Model response status

Since the HTTP Response status code isn't in the headers dictionary it
should be modeled separately.

Fixes #24

tests/test_warcdb.py

@@ -89,3 +89,15 @@ def test_http_header():
  "value": "Wget/1.21.3",
  "warc_record_id": "<urn:uuid:6E9096E2-5D54-4CD6-A157-1DE4A7040DEB>",
  } in req_headers
+
+
+def test_http_header():
+ runner = CliRunner()
+ runner.invoke(
+ warcdb_cli, ["import", db_file, str(pathlib.Path("tests/google.warc"))]
+ )
+ db = sqlite_utils.Database(db_file)
+ responses = db["response"].rows
+ assert next(responses)["http_status"] == 301
+ assert next(responses)["http_status"] == 302
+ assert next(responses)["http_status"] == 200

warcdb/__init__.py

@@ -182,6 +182,8 @@ def __iadd__(self, r: ArcWarcRecord):
  )
 
  elif r.rec_type == "response":
+ if r.http_headers:
+ record_dict["http_status"] = r.http_headers.get_statuscode()
  self.db.table("response").insert(
  record_dict,
  pk="warc_record_id",

warcdb/migrations.py

@@ -119,3 +119,8 @@ def m002_headers(db):
  FROM response, JSON_EACH(response.http_headers) AS header
  """,
  )
+
+
+@migration()
+def m003_status(db):
+ db["response"].add_column("http_status", int)