From bbf2c124fba16be48640dc910081dd226abe810d Mon Sep 17 00:00:00 2001 From: Jashandeep Sohi Date: Mon, 21 Sep 2015 16:27:01 -0700 Subject: [PATCH 1/6] Use a mmap file to read data in StaticRoute when using the sendfile fallback. --- aiohttp/web_urldispatcher.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py index 62851b6edca..d418cfe384f 100644 --- a/aiohttp/web_urldispatcher.py +++ b/aiohttp/web_urldispatcher.py @@ -6,6 +6,7 @@ import re import os import inspect +import mmap from urllib.parse import urlencode, unquote @@ -235,14 +236,17 @@ def _sendfile_fallback(self, req, resp, fobj, count): transferred in chunks controlled by the `chunk_size` argument to :class:`StaticRoute`. """ + f_mm = mmap.mmap(fobj.fileno(), 0, access=mmap.ACCESS_READ) chunk_size = self._chunk_size - chunk = fobj.read(chunk_size) + chunk = f_mm.read(chunk_size) while chunk and count > chunk_size: resp.write(chunk) yield from resp.drain() count = count - chunk_size - chunk = fobj.read(chunk_size) + chunk = f_mm.read(chunk_size) + + f_mm.close() if chunk: resp.write(chunk[:count]) From f1858b21409c96c74146feda5c63b31c8e968b3e Mon Sep 17 00:00:00 2001 From: Jashandeep Sohi Date: Tue, 22 Sep 2015 18:05:02 -0700 Subject: [PATCH 2/6] explicitly check if static file is empty & return empty response early --- aiohttp/web_urldispatcher.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py index d418cfe384f..31ea597c00b 100644 --- a/aiohttp/web_urldispatcher.py +++ b/aiohttp/web_urldispatcher.py @@ -285,6 +285,9 @@ def handle(self, request): file_size = st.st_size resp.content_length = file_size + if file_size == 0: + return resp + resp.start(request) with open(filepath, 'rb') as f: From a20c956705235f1fb74b488119fefc2d67d71e62 Mon Sep 17 00:00:00 2001 From: Jashandeep Sohi Date: Tue, 22 Sep 2015 18:07:07 -0700 Subject: [PATCH 3/6] Add an empty static file test --- tests/empty.dat | 0 tests/test_web_functional.py | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 tests/empty.dat diff --git a/tests/empty.dat b/tests/empty.dat new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index 596587d587f..6f900fd51e0 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -1030,6 +1030,26 @@ def go(dirname, filename): self.loop.run_until_complete(go(here, filename)) + def test_static_file_empty(self): + + @asyncio.coroutine + def go(dirname, filename): + app, _, url = yield from self.create_server( + 'GET', '/static/' + filename + ) + app.router.add_static('/static', dirname) + + resp = yield from request('GET', url, loop=self.loop) + self.assertEqual(200, resp.status) + txt = yield from resp.text() + self.assertEqual('', txt) + self.assertEqual('0', resp.headers['CONTENT-LENGTH']) + resp.close() + + here = os.path.dirname(__file__) + filename = 'empty.dat' + self.loop.run_until_complete(go(here, filename)) + class TestStaticFileSendfileFallback(StaticFileMixin, unittest.TestCase): From 34ea77938849c7f4e9a68ce8754fa6dfd0724675 Mon Sep 17 00:00:00 2001 From: Jashandeep Sohi Date: Sun, 27 Sep 2015 18:49:51 -0700 Subject: [PATCH 4/6] Refactor `_sendfile_fallback` to use slice notation to read chunks from mmap --- aiohttp/web_urldispatcher.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py index 31ea597c00b..1f1ab8e80b0 100644 --- a/aiohttp/web_urldispatcher.py +++ b/aiohttp/web_urldispatcher.py @@ -236,22 +236,19 @@ def _sendfile_fallback(self, req, resp, fobj, count): transferred in chunks controlled by the `chunk_size` argument to :class:`StaticRoute`. """ - f_mm = mmap.mmap(fobj.fileno(), 0, access=mmap.ACCESS_READ) + f_mm = mmap.mmap(fobj.fileno(), count, access=mmap.ACCESS_READ) chunk_size = self._chunk_size - chunk = f_mm.read(chunk_size) - while chunk and count > chunk_size: - resp.write(chunk) + chunk_starts = range(0, count, chunk_size) + chunk_stops = range(min(count, chunk_size), + count+chunk_size, chunk_size) + + for i, j in zip(chunk_starts, chunk_stops): + resp.write(f_mm[i:j]) yield from resp.drain() - count = count - chunk_size - chunk = f_mm.read(chunk_size) f_mm.close() - if chunk: - resp.write(chunk[:count]) - yield from resp.drain() - if hasattr(os, "sendfile"): # pragma: no cover _sendfile = _sendfile_system else: # pragma: no cover From cf8678cfdb8a217b1bbd5aaad05b4518808d2d71 Mon Sep 17 00:00:00 2001 From: Jashandeep Sohi Date: Tue, 29 Sep 2015 14:37:09 -0700 Subject: [PATCH 5/6] Remove unnecessary min calculation for `chunk_stops` --- aiohttp/web_urldispatcher.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py index 1f1ab8e80b0..55f2df49617 100644 --- a/aiohttp/web_urldispatcher.py +++ b/aiohttp/web_urldispatcher.py @@ -240,8 +240,7 @@ def _sendfile_fallback(self, req, resp, fobj, count): chunk_size = self._chunk_size chunk_starts = range(0, count, chunk_size) - chunk_stops = range(min(count, chunk_size), - count+chunk_size, chunk_size) + chunk_stops = range(count, count+chunk_size, chunk_size) for i, j in zip(chunk_starts, chunk_stops): resp.write(f_mm[i:j]) From 3c681b261a6ea669805e284911faa68b22207b75 Mon Sep 17 00:00:00 2001 From: Jashandeep Sohi Date: Tue, 29 Sep 2015 18:17:40 -0700 Subject: [PATCH 6/6] Fix `chunk_stops` starting point; should be chunk_size not count --- aiohttp/web_urldispatcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py index 55f2df49617..9746629a298 100644 --- a/aiohttp/web_urldispatcher.py +++ b/aiohttp/web_urldispatcher.py @@ -240,7 +240,7 @@ def _sendfile_fallback(self, req, resp, fobj, count): chunk_size = self._chunk_size chunk_starts = range(0, count, chunk_size) - chunk_stops = range(count, count+chunk_size, chunk_size) + chunk_stops = range(chunk_size, count+chunk_size, chunk_size) for i, j in zip(chunk_starts, chunk_stops): resp.write(f_mm[i:j])