diff --git a/README.md b/README.md index 31188e7..0c4a8ab 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,9 @@ Here's the relational schema of the `.warcdb` file. ![WarcDB Schema](schema.png) +In addition to the core tables that map to the WARC record types there are also helper views that make it a bit easier to query data: + +- *http_header*: A view of HTTP headers in responses where each row is a tuple of `(warc_record_id, name, value)` ## Motivation diff --git a/tests/test_warcdb.py b/tests/test_warcdb.py index bc6b41c..dc2be10 100644 --- a/tests/test_warcdb.py +++ b/tests/test_warcdb.py @@ -64,3 +64,19 @@ def test_column_names(): assert re.match(r"^[a-z_]+", col.name), f"column {col.name} named correctly" os.remove(db_file) + + +def test_http_header(): + runner = CliRunner() + runner.invoke( + warcdb_cli, ["import", db_file, str(pathlib.Path("tests/google.warc"))] + ) + + db = sqlite_utils.Database(db_file) + headers = list(db["http_header"].rows) + assert len(headers) == 43 + assert { + "name": "content-type", + "value": "text/html; charset=UTF-8", + "warc_record_id": "", + } in headers diff --git a/warcdb/migrations.py b/warcdb/migrations.py index 64925ba..f5f2582 100644 --- a/warcdb/migrations.py +++ b/warcdb/migrations.py @@ -95,3 +95,17 @@ def m001_initial(db): ("warc_concurrent_to", "metadata", "warc_record_id"), ], ) + + +@migration() +def m002_headers(db): + db.create_view( + "http_header", + """ + SELECT + response.warc_record_id AS warc_record_id, + LOWER(JSON_EXTRACT(JSON_EACH.VALUE, '$.header')) AS name, + JSON_EXTRACT(JSON_EACH.VALUE, '$.value') AS value + FROM response, JSON_EACH(response.http_headers) + """, + )