Skip to content

Commit

Permalink
Fixed get_paths not auto-paging (#16581)
Browse files Browse the repository at this point in the history 
* Fixed get_paths not auto-paging

* fixed tests

* rerecorded test failures
  • Loading branch information
@tasherif-msft
tasherif-msft authored Feb 8, 2021
1 parent 6bc937d commit 1b9e576
Show file tree
Hide file tree
Showing 8 changed files with 1,380 additions and 160 deletions.
21 changes: 15 additions & 6 deletions ...-datalake/azure/storage/filedatalake/_generated/aio/operations/_file_system_operations.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,8 @@ def list_paths(
error_map.update(kwargs.pop('error_map', {}))
accept = "application/json"

def prepare_request(next_link=None):
# TODO: change this once continuation/next_link autorest PR is merged
def prepare_request(next_link=None, cont_token=None):
# Construct headers
header_parameters = {} # type: Dict[str, Any]
if request_id_parameter is not None:
Expand All @@ -458,8 +459,9 @@ def prepare_request(next_link=None):
query_parameters['resource'] = self._serialize.query("self._config.resource", self._config.resource, 'str')
if timeout is not None:
query_parameters['timeout'] = self._serialize.query("timeout", timeout, 'int', minimum=0)
if continuation is not None:
query_parameters['continuation'] = self._serialize.query("continuation", continuation, 'str')
# TODO: change this once continuation/next_link autorest PR is merged
if cont_token is not None:
query_parameters['continuation'] = self._serialize.query("continuation", cont_token, 'str')
if path is not None:
query_parameters['directory'] = self._serialize.query("path", path, 'str')
query_parameters['recursive'] = self._serialize.query("recursive", recursive, 'bool')
Expand All @@ -480,14 +482,21 @@ def prepare_request(next_link=None):
return request

async def extract_data(pipeline_response):
# TODO: change this once continuation/next_link autorest PR is merged
try:
cont_token = pipeline_response.http_response.headers['x-ms-continuation']
except KeyError:
cont_token = None
deserialized = self._deserialize('PathList', pipeline_response)
list_of_elem = deserialized.paths
if cls:
list_of_elem = cls(list_of_elem)
return None, AsyncList(list_of_elem)
return cont_token, AsyncList(list_of_elem)

async def get_next(next_link=None):
request = prepare_request(next_link)
# TODO: change this once continuation/next_link autorest PR is merged
async def get_next(cont_token=None):
cont_token = cont_token if not continuation else continuation
request = prepare_request(cont_token=cont_token)

pipeline_response = await self._client._pipeline.run(request, stream=False, **kwargs)
response = pipeline_response.http_response
Expand Down
26 changes: 18 additions & 8 deletions ...file-datalake/azure/storage/filedatalake/_generated/operations/_file_system_operations.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ def set_properties(
401: ClientAuthenticationError, 404: ResourceNotFoundError, 409: ResourceExistsError
}
error_map.update(kwargs.pop('error_map', {}))

_if_modified_since = None
_if_unmodified_since = None
if modified_access_conditions is not None:
Expand Down Expand Up @@ -340,7 +340,7 @@ def delete(
401: ClientAuthenticationError, 404: ResourceNotFoundError, 409: ResourceExistsError
}
error_map.update(kwargs.pop('error_map', {}))

_if_modified_since = None
_if_unmodified_since = None
if modified_access_conditions is not None:
Expand Down Expand Up @@ -447,7 +447,8 @@ def list_paths(
error_map.update(kwargs.pop('error_map', {}))
accept = "application/json"

def prepare_request(next_link=None):
# TODO: change this once continuation/next_link autorest PR is merged
def prepare_request(next_link=None, cont_token=None):
# Construct headers
header_parameters = {} # type: Dict[str, Any]
if request_id_parameter is not None:
Expand All @@ -467,8 +468,9 @@ def prepare_request(next_link=None):
query_parameters['resource'] = self._serialize.query("self._config.resource", self._config.resource, 'str')
if timeout is not None:
query_parameters['timeout'] = self._serialize.query("timeout", timeout, 'int', minimum=0)
if continuation is not None:
query_parameters['continuation'] = self._serialize.query("continuation", continuation, 'str')
# TODO: change this once continuation/next_link autorest PR is merged
if cont_token is not None:
query_parameters['continuation'] = self._serialize.query("continuation", cont_token, 'str')
if path is not None:
query_parameters['directory'] = self._serialize.query("path", path, 'str')
query_parameters['recursive'] = self._serialize.query("recursive", recursive, 'bool')
Expand All @@ -489,14 +491,22 @@ def prepare_request(next_link=None):
return request

def extract_data(pipeline_response):
# TODO: change this once continuation/next_link autorest PR is merged
try:
cont_token = pipeline_response.http_response.headers['x-ms-continuation']
except KeyError:
cont_token = None
deserialized = self._deserialize('PathList', pipeline_response)
list_of_elem = deserialized.paths
if cls:
list_of_elem = cls(list_of_elem)
return None, iter(list_of_elem)
# TODO: change this once continuation/next_link autorest PR is merged
return cont_token, iter(list_of_elem)

def get_next(next_link=None):
request = prepare_request(next_link)
# TODO: change this once continuation/next_link autorest PR is merged
def get_next(cont_token=None):
cont_token = cont_token if not continuation else continuation
request = prepare_request(cont_token=cont_token)

pipeline_response = self._client._pipeline.run(request, stream=False, **kwargs)
response = pipeline_response.http_response
Expand Down
Loading

0 comments on commit 1b9e576

Please sign in to comment.