Merge pull request #868 from sirosen/add-ls-params

Add support for limit and offset to ls

changelog.d/20231016_114115_sirosen_add_ls_params.rst

@@ -0,0 +1,5 @@
+Added
+~~~~~
+
+- ``TransferClient.operation_ls`` now supports the ``limit`` and ``offset``
+  parameters (:pr:`NUMBER`)

src/globus_sdk/services/transfer/client.py

@@ -1210,6 +1210,8 @@ def operation_ls(
         *,
         show_hidden: bool | None = None,
         orderby: str | list[str] | None = None,
+        limit: int | None = None,
+        offset: int | None = None,
         # note: filter is a soft keyword in python, so using this name is okay
         # pylint: disable=redefined-builtin
         filter: (
@@ -1226,6 +1228,11 @@ def operation_ls(
         :param show_hidden: Show hidden files (names beginning in dot).
             Defaults to true.
         :type show_hidden: bool, optional
+        :param limit: Limit the number of results returned. Defaults to 100,000 ,
+            which is also the maximum.
+        :type limit: int, optional
+        :param offset: Offset into the result list, which can be used to page results.
+        :type offset: int, optional
         :param orderby: One or more order-by options. Each option is
             either a field name or a field name followed by a space and 'ASC' or 'DESC'
             for ascending or descending.
@@ -1243,6 +1250,17 @@ def operation_ls(
         :param query_params: Additional passthrough query parameters
         :type query_params: dict, optional
 
+        .. note::
+
+            Pagination is not supported by the GridFTP protocol, and therefore
+            limit+offset pagination will result in the Transfer service repeatedly
+            fetching an entire directory listing from the server and filtering it before
+            returning it to the client.
+
+            For latency-sensitive applications, such usage may still be more efficient
+            than asking for a very large directory listing as it reduces the size of the
+            payload passed between the Transfer service and the client.
+
         .. tab-set::
 
             .. tab-item:: Example Usage
@@ -1289,6 +1307,10 @@ def operation_ls(
             query_params["path"] = path
         if show_hidden is not None:
             query_params["show_hidden"] = 1 if show_hidden else 0
+        if limit is not None:
+            query_params["limit"] = limit
+        if offset is not None:
+            query_params["offset"] = offset
         if orderby is not None:
             if isinstance(orderby, str):
                 query_params["orderby"] = orderby

tests/functional/services/transfer/test_operation_ls.py

@@ -0,0 +1,95 @@
+import urllib.parse
+
+import pytest
+
+from globus_sdk._testing import RegisteredResponse, get_last_request, load_response
+from tests.common import GO_EP1_ID
+
+
+def _mk_item(*, name, typ, size=0):
+    return {
+        "DATA_TYPE": "file",
+        "group": "tutorial",
+        "last_modified": "2018-04-04 18:30:26+00:00",
+        "link_group": None,
+        "link_last_modified": None,
+        "link_size": None,
+        "link_target": None,
+        "link_user": None,
+        "name": name,
+        "permissions": "0755" if typ == "dir" else "0644",
+        "size": 4096 if typ == "dir" else size,
+        "type": typ,
+        "user": "snork",
+    }
+
+
+def _mk_ls_data():
+    return {
+        "DATA": [
+            _mk_item(name="foo", typ="dir"),
+            _mk_item(name="tempdir1", typ="dir"),
+            _mk_item(name=".bashrc", typ="file", size=3771),
+            _mk_item(name=".profile", typ="file", size=807),
+        ]
+    }
+
+
+@pytest.fixture(autouse=True)
+def _setup_ls_response():
+    load_response(
+        RegisteredResponse(
+            service="transfer",
+            path=f"/operation/endpoint/{GO_EP1_ID}/ls",
+            json=_mk_ls_data(),
+        ),
+    )
+
+
+def test_operation_ls(client):
+    ls_path = f"https://transfer.api.globus.org/v0.10/operation/endpoint/{GO_EP1_ID}/ls"
+
+    # load the tutorial endpoint ls doc
+    ls_doc = client.operation_ls(GO_EP1_ID)
+
+    # check that the result is an iterable of file and dir dict objects
+    for x in ls_doc:
+        assert "DATA_TYPE" in x
+        assert x["DATA_TYPE"] in ("file", "dir")
+
+    req = get_last_request()
+    assert req.url == ls_path
+
+
+@pytest.mark.parametrize(
+    "kwargs, expected_qs",
+    [
+        # orderby with a single str
+        ({"orderby": "name"}, {"orderby": ["name"]}),
+        # orderby with a multiple strs
+        (
+            {"orderby": ["size DESC", "name", "type"]},
+            {"orderby": ["size DESC,name,type"]},
+        ),
+        # orderby + filter
+        (
+            {"orderby": "name", "filter": "name:~*.png"},
+            {"orderby": ["name"], "filter": ["name:~*.png"]},
+        ),
+        # local_user
+        (
+            {"local_user": "my-user"},
+            {"local_user": ["my-user"]},
+        ),
+        # limit+offset
+        (
+            {"limit": 10, "offset": 5},
+            {"limit": ["10"], "offset": ["5"]},
+        ),
+    ],
+)
+def test_operation_ls_params(client, kwargs, expected_qs):
+    client.operation_ls(GO_EP1_ID, **kwargs)
+    req = get_last_request()
+    parsed_qs = urllib.parse.parse_qs(urllib.parse.urlparse(req.url).query)
+    assert parsed_qs == expected_qs

tests/functional/services/transfer/test_simple.py

@@ -100,60 +100,6 @@ def test_create_endpoint_invalid_activation_servers(client):
     assert "either MyProxy or OAuth, not both" in str(excinfo.value)
 
 
-def test_operation_ls(client):
-    """
-    Does an `ls` on go#ep1, validate results, and check that the request parameters were
-    formatted and sent correctly.
-    """
-    # register get_endpoint mock data
-    register_api_route_fixture_file(
-        "transfer", f"/operation/endpoint/{GO_EP1_ID}/ls", "operation_ls_goep1.json"
-    )
-    ls_path = f"https://transfer.api.globus.org/v0.10/operation/endpoint/{GO_EP1_ID}/ls"
-
-    # load the tutorial endpoint ls doc
-    ls_doc = client.operation_ls(GO_EP1_ID)
-
-    # check that the result is an iterable of file and dir dict objects
-    count = 0
-    for x in ls_doc:
-        assert "DATA_TYPE" in x
-        assert x["DATA_TYPE"] in ("file", "dir")
-        count += 1
-    # not exact, just make sure the fixture wasn't empty
-    assert count > 3
-
-    req = get_last_request()
-    assert req.url == ls_path
-
-    # don't change the registered response
-    # the resulting data might be "wrong", but we're just checking request formatting
-
-    # orderby with a single str
-    client.operation_ls(GO_EP1_ID, orderby="name")
-    req = get_last_request()
-    parsed_qs = urllib.parse.parse_qs(urllib.parse.urlparse(req.url).query)
-    assert parsed_qs == {"orderby": ["name"]}
-
-    # orderby multiple strs
-    client.operation_ls(GO_EP1_ID, orderby=["size DESC", "name", "type"])
-    req = get_last_request()
-    parsed_qs = urllib.parse.parse_qs(urllib.parse.urlparse(req.url).query)
-    assert parsed_qs == {"orderby": ["size DESC,name,type"]}
-
-    # orderby + filter
-    client.operation_ls(GO_EP1_ID, orderby="name", filter="name:~*.png")
-    req = get_last_request()
-    parsed_qs = urllib.parse.parse_qs(urllib.parse.urlparse(req.url).query)
-    assert parsed_qs == {"orderby": ["name"], "filter": ["name:~*.png"]}
-
-    # local_user
-    client.operation_ls(GO_EP1_ID, local_user="my-user")
-    req = get_last_request()
-    parsed_qs = urllib.parse.parse_qs(urllib.parse.urlparse(req.url).query)
-    assert parsed_qs == {"local_user": ["my-user"]}
-
-
 def test_autoactivation(client):
     """
     Do `autoactivate` on go#ep1, validate results, and check that `if_expires_in` can be