[http] fix parsing link header

visidata/loaders/http.py

@@ -1,3 +1,5 @@
+import re
+
 from visidata import Path, RepeatFile, vd, VisiData
 from visidata.loaders.tsv import splitter
 
@@ -47,24 +49,59 @@ def _iter_lines(path=path, response=response, max_next=vd.options.http_max_next)
             linkhdr = response.getheader('Link')
             src = None
             if linkhdr:
-                links = urllib.parse.parse_header(linkhdr)
-                src = links.get('next', {}).get('url', None)
+                links = parse_header_links(linkhdr)
+                link_data = {}
+                for link in links:
+                    key = link.get('rel') or link.get('url')
+                    link_data[key] = link
+                src = link_data.get('next', {}).get('url', None)
 
             if not src:
                 break
 
             n += 1
             if n > max_next:
-                vd.warning(f'stopping at max {max_next} pages')
+                vd.warning(f'stopping at max next pages: {max_next} pages')
                 break
 
             vd.status(f'fetching next page from {src}')
-            response = requests.get(src, stream=True, **vd.options.getall('http_req_'))
+            req = urllib.request.Request(src, **vd.options.getall('http_req_'))
+            response = urllib.request.urlopen(req)
 
     # add resettable iterator over contents as an already-open fp
     path.fptext = RepeatFile(_iter_lines())
 
     return vd.openSource(path, filetype=filetype)
 
+def parse_header_links(link_header):
+    '''Return a list of dictionaries:
+    [{'url': 'https://example.com/content?page=1', 'rel': 'prev'},
+     {'url': 'https://example.com/content?page=3', 'rel': 'next'}]
+    Takes a link header string, of the form
+    '<https://example.com/content?page=1>; rel="prev", <https://example.com/content?page=3>; rel="next"'
+    See https://datatracker.ietf.org/doc/html/rfc8288#section-3
+    '''
+
+    links = []
+    quote_space = ' \'"'
+    link_header = link_header.strip(quote_space)
+    if not link_header: return []
+    for link_value in re.split(', *<', link_header):
+        if ';' in link_value:
+            url, params = link_value.split(';', maxsplit=1)
+        else:
+            url, params = link_value, ''
+        link = {'url': url.strip('<>' + quote_space)}
+
+        for param in params.split(';'):
+            if '=' in param:
+                key, value = param.split('=')
+                key = key.strip(quote_space)
+                value = value.strip(quote_space)
+                link[key] = value
+            else:
+                break
+        links.append(link)
+    return links
 
 VisiData.openurl_https = VisiData.openurl_http

visidata/plugins.py

@@ -5,7 +5,7 @@
 import shutil
 import importlib
 import subprocess
-import urllib
+import urllib.error
 
 from visidata import VisiData, vd, Path, CellColorizer, JsonLinesSheet, AttrDict, Column, Progress, ExpectedException, BaseSheet, asyncsingle, asyncthread