Show requests/response in log on failure

conftest.py

@@ -167,6 +167,7 @@ def _item_is_skipped(item):
 
 
 def pytest_collection_finish(session):
+    from utils import weblog
 
     if session.config.option.collectonly:
         return
@@ -200,6 +201,7 @@ def pytest_collection_finish(session):
         setup_method = getattr(item.instance, setup_method_name)
         logger.debug(f"Call {setup_method} for {item}")
         try:
+            weblog.current_nodeid = item.nodeid
             setup_method()
         except Exception:
             logger.exception("Unexpected failure during setup method call")
@@ -208,12 +210,23 @@ def pytest_collection_finish(session):
             raise
         else:
             terminal.write(".", bold=True, green=True)
+        finally:
+            weblog.current_nodeid = None
 
     terminal.write("\n\n")
 
     context.scenario.post_setup(session)
 
 
+def pytest_runtest_call(item):
+    from utils import weblog
+
+    if item.nodeid in weblog.responses:
+        for response in weblog.responses[item.nodeid]:
+            request = response["request"]
+            logger.info(f"weblog {request['method']} {request['url']} -> {response['status_code']}")
+
+
 def pytest_json_modifyreport(json_report):
 
     try:

utils/_context/_scenarios.py

@@ -425,6 +425,13 @@ def _wait_interface(interface, session, timeout):
 
         interface.wait(timeout)
 
+    def close_targets(self):
+        from utils import weblog
+
+        super().close_targets()
+
+        weblog.save_requests(self.host_log_folder)
+
     @property
     def dd_site(self):
         return self.agent_container.dd_site

utils/_weblog.py

@@ -2,10 +2,12 @@
 # This product includes software developed at Datadog (https://www.datadoghq.com/).
 # Copyright 2021 Datadog, Inc.
 
-import urllib
+from collections import defaultdict
+import json
 import os
 import random
 import string
+import urllib
 
 import requests
 import grpc
@@ -45,6 +47,16 @@ def __init__(self):
         else:
             self.domain = "localhost"
 
+        self.responses = defaultdict(list)
+        self.current_nodeid = None  # will be used to store request made by a given nodeid
+
+    def save_requests(self, log_folder):
+        try:
+            with open(f"{log_folder}/weblog_responses.json", "w", encoding="utf-8") as f:
+                json.dump(dict(self.responses), f, indent=2)
+        except:
+            logger.exception("Can't save responses log")
+
     def get(self, path="/", params=None, headers=None, cookies=None, **kwargs):
         return self.request("GET", path, params=params, headers=headers, cookies=cookies, **kwargs)
 
@@ -98,6 +110,13 @@ def request(
 
         logger.debug(f"Request {rid}: {r.status_code}")
 
+        self.responses[self.current_nodeid].append(
+            {
+                "request": {"method": method, "url": url, "headers": headers, "params": params, "data": data},
+                "status_code": r.status_code,
+            }
+        )
+
         return r
 
     def _get_url(self, path, domain, port, query=None):