Modified view names, add request headers and doc.

Add a similar table for HTTP requests. Prefix the view names with a `v_` to distinguish it in the schema from
actual tables.

Also add a description of the view with a table that defines the columns.

README.md

@@ -52,9 +52,29 @@ Here's the relational schema of the `.warcdb` file.
 
 ![WarcDB Schema](schema.png)
 
-In addition to the core tables that map to the WARC record types there are also helper views that make it a bit easier to query data:
+### Views
 
-- *http_header*: A view of HTTP headers in responses where each row is a tuple of `(warc_record_id, name, value)` 
+In addition to the core tables that map to the WARC record types there are also helper *views* that make it a bit easier to query data:
+
+#### v_request_http_header
+
+A view of HTTP headers in WARC request records:
+
+| Column Name | Column Type | Description |
+| -------------- | ----------- | ---------------------------------------------------------------------- |
+| warc_record_id | text | The WARC-Record-Id for the *request* record that it was extracted from. |
+| name | text | The lowercased HTTP header name (e.g. content-type) |
+| value | text | The HTTP header value (e.g. text/html) |
+
+#### v_response_http_header
+
+A view of HTTP headers in WARC response records:
+
+| Column Name | Column Type | Description |
+| -------------- | ----------- | ---------------------------------------------------------------------- |
+| warc_record_id | text | The WARC-Record-Id for the *response* record that it was extracted from. |
+| name | text | The lowercased HTTP header name (e.g. content-type) |
+| value | text | The HTTP header value (e.g. text/html) |
 
 ## Motivation
 

tests/test_warcdb.py

@@ -73,10 +73,21 @@ def test_http_header():
  )
 
  db = sqlite_utils.Database(db_file)
- headers = list(db["http_header"].rows)
- assert len(headers) == 43
+
+ resp_headers = list(db["v_response_http_header"].rows)
+ assert len(resp_headers) == 43
  assert {
  "name": "content-type",
  "value": "text/html; charset=UTF-8",
  "warc_record_id": "<urn:uuid:2008CBED-030B-435B-A4DF-09A842DDB764>",
- } in headers
+ } in resp_headers
+
+ req_headers = list(db["v_request_http_header"].rows)
+ assert len(req_headers) == 17
+ assert {
+ "name": "user-agent",
+ "value": "Wget/1.21.3",
+ "warc_record_id": "<urn:uuid:6E9096E2-5D54-4CD6-A157-1DE4A7040DEB>"
+ } in req_headers
+
+

warcdb/migrations.py

@@ -100,7 +100,17 @@ def m001_initial(db):
 @migration()
 def m002_headers(db):
  db.create_view(
- "http_header",
+ "v_request_http_header",
+ """
+ SELECT
+ request.warc_record_id AS warc_record_id,
+ LOWER(JSON_EXTRACT(header.VALUE, '$.header')) AS name,
+ JSON_EXTRACT(header.VALUE, '$.value') AS value
+ FROM request, JSON_EACH(request.http_headers) AS header
+ """,
+ )
+ db.create_view(
+ "v_response_http_header",
  """
  SELECT
  response.warc_record_id AS warc_record_id,