User-Agent has a default and is used for check_url

openzim · Oct 23, 2023 · 028a4dd · 028a4dd
1 parent f22bb92
commit 028a4dd
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Scraper fails for all HTTP error codes returned when checking URL at startup (#223)
+- User-Agent now has a default value + manipulation of spaces with suffix and  (#228)
+- Same User-Agent is used for check_url and browsertrix crawler (#227)
 
 ## [1.5.3] - 2023-10-02
 

diff --git a/test/integration.py b/test/integration.py
@@ -36,7 +36,7 @@ def test_user_agent():
                     print(record.http_headers)
                     ua = record.http_headers.get_header("User-Agent")
                     if ua:
-                        assert "Pixel" in ua
+                        assert "Mozilla" in ua
                         assert ua.endswith(" +Zimit test@example.com")
                         found = True
 

diff --git a/zimit.py b/zimit.py
@@ -226,14 +226,15 @@ def zimit(args=None):
 
     parser.add_argument(
         "--userAgent",
-        help="Override user-agent with specified",
+        help="Override default user-agent with specified value ; --userAgentSuffix is still applied",
+        default="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15"
     )
 
     parser.add_argument(
         "--userAgentSuffix",
         help="Append suffix to existing browser user-agent "
         "(ex: +MyCrawler, info@example.com)",
-        default="+Zimit ",
+        default="+Zimit",
     )
 
     parser.add_argument(
@@ -344,8 +345,14 @@ def zimit(args=None):
 
     url = zimit_args.url
 
+    user_agent = zimit_args.userAgent
+    if zimit_args.userAgentSuffix:
+        user_agent += f" {zimit_args.userAgentSuffix}"
+    if zimit_args.adminEmail:
+        user_agent += f" {zimit_args.adminEmail}"
+
     if url:
-        url = check_url(url, zimit_args.scopeType)
+        url = check_url(url, user_agent, zimit_args.scopeType)
         warc2zim_args.append("--url")
         warc2zim_args.append(url)
 
@@ -394,12 +401,8 @@ def cleanup():
         cmd_args.append("--url")
         cmd_args.append(url)
 
-    user_agent_suffix = zimit_args.userAgentSuffix
-    if zimit_args.adminEmail:
-        user_agent_suffix += zimit_args.adminEmail
-
-    cmd_args.append("--userAgentSuffix")
-    cmd_args.append(user_agent_suffix)
+    cmd_args.append("--userAgent")
+    cmd_args.append(user_agent)
 
     cmd_args.append("--cwd")
     cmd_args.append(str(temp_root_dir))
@@ -445,11 +448,11 @@ def cleanup():
     return warc2zim(warc2zim_args)
 
 
-def check_url(url, scope=None):
+def check_url(url, user_agent, scope=None):
     url = urllib.parse.urlparse(url)
     try:
         resp = requests.head(
-            url.geturl(), stream=True, allow_redirects=True, timeout=(12.2, 27)
+            url.geturl(), stream=True, allow_redirects=True, timeout=(12.2, 27), headers={"User-Agent": user_agent}
         )
         resp.raise_for_status()
     except requests.exceptions.RequestException as exc:
@@ -505,7 +508,6 @@ def get_node_cmd_line(args):
         "allowHashUrls",
         "lang",
         "mobileDevice",
-        "userAgent",
         "useSitemap",
         "behaviors",
         "behaviorTimeout",