Merge pull request #79 from w3c/jgraham/file_chunks

Add ability to write files in chunks.

wptserve/handlers.py

@@ -140,6 +140,7 @@ def __call__(self, request, response):
                 ml_extensions = {".html", ".htm", ".xht", ".xhtml", ".xml", ".svg"}
                 escape_type = "html" if os.path.splitext(path)[1] in ml_extensions else "none"
                 pipeline = Pipeline("sub(%s)" % escape_type)
+
             if pipeline is not None:
                 response = pipeline(request, response)
 
@@ -174,10 +175,12 @@ def load_headers(self, request, path):
                     for line in data.splitlines() if line]
 
     def get_data(self, response, path, byte_ranges):
-        with open(path, 'rb') as f:
-            if byte_ranges is None:
-                return f.read()
-            else:
+        """Return either the handle to a file, or a string containing
+        the content of a chunk of the file, if we have a range request."""
+        if byte_ranges is None:
+            return open(path, 'rb')
+        else:
+            with open(path, 'rb') as f:
                 response.status = 206
                 if len(byte_ranges) > 1:
                     parts_content_type, content = self.set_response_multipart(response,

wptserve/pipes.py

@@ -8,7 +8,7 @@
 
 
 def resolve_content(response):
-    rv = "".join(item for item in response.iter_content())
+    rv = "".join(item for item in response.iter_content(read_file=True))
     if type(rv) == unicode:
         rv = rv.encode(response.encoding)
     return rv

wptserve/response.py

@@ -168,13 +168,25 @@ def delete_cookie(self, name, path="/", domain=None):
         self.set_cookie(name, None, path=path, domain=domain, max_age=0,
                         expires=timedelta(days=-1))
 
-    def iter_content(self):
+    def iter_content(self, read_file=False):
         """Iterator returning chunks of response body content.
 
         If any part of the content is a function, this will be called
-        and the resulting value (if any) returned."""
-        if type(self.content) in types.StringTypes:
+        and the resulting value (if any) returned.
+
+        :param read_file: - boolean controlling the behaviour when content
+        is a file handle. When set to False the handle will be returned directly
+        allowing the file to be passed to the output in small chunks. When set to
+        True, the entire content of the file will be returned as a string facilitating
+        non-streaming operations like template substitution.
+        """
+        if isinstance(self.content, types.StringTypes):
             yield self.content
+        elif hasattr(self.content, "read"):
+            if read_file:
+                yield self.content.read()
+            else:
+                yield self.content
         else:
             for item in self.content:
                 if hasattr(item, "__call__"):
@@ -355,6 +367,7 @@ def __init__(self, handler, response):
         self._headers_complete = False
         self.content_written = False
         self.request = response.request
+        self.file_chunk_size = 32 * 1024
 
     def write_status(self, code, message=None):
         """Write out the status line of a response.
@@ -411,7 +424,10 @@ def end_headers(self):
 
     def write_content(self, data):
         """Write the body of the response."""
-        self.write(self.encode(data))
+        if isinstance(data, types.StringTypes):
+            self.write(data)
+        else:
+            self.write_content_file(data)
         if not self._response.explicit_flush:
             self.flush()
 
@@ -425,6 +441,19 @@ def write(self, data):
             # This can happen if the socket got closed by the remote end
             pass
 
+    def write_content_file(self, data):
+        """Write a file-like object directly to the response in chunks.
+        Does not flush."""
+        self.content_written = True
+        while True:
+            buf = data.read(self.file_chunk_size)
+            if not buf:
+                break
+            try:
+                self._wfile.write(buf)
+            except socket.error:
+                break
+
     def encode(self, data):
         """Convert unicode to bytes according to response.encoding."""
         if isinstance(data, str):