From eb10a5354485707c4825b7807b5d6e0236d87e4a Mon Sep 17 00:00:00 2001 From: jakirkham Date: Tue, 21 Jul 2020 17:01:51 -0700 Subject: [PATCH] Use `memoryview` in `unpack_frames` (#3980) * Minor whitespace adjustment * Coerce input to `memoryview` in `unpack_frames` Selecting out each frame from the input causes a copy, which increases memory usage and slows down `unpack_frames`. To fix this, coerce the input to a `memoryview`. This way slices into the `memoryview` only take a view onto the underlying data, which is quite fast and doesn't result in additional memory usage. --- distributed/protocol/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py index fd50b6ca09..8dc75a28c6 100644 --- a/distributed/protocol/utils.py +++ b/distributed/protocol/utils.py @@ -120,8 +120,11 @@ def unpack_frames(b): -------- pack_frames """ + b = memoryview(b) + fmt = "Q" fmt_size = struct.calcsize(fmt) + (n_frames,) = struct.unpack_from(fmt, b) lengths = struct.unpack_from(f"{n_frames}{fmt}", b, fmt_size)