docs: Demonstrate uploading and receiving files

demos/file_upload/file_receiver.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+
+"""Usage: python file_receiver.py
+
+Demonstrates a server that receives a multipart-form-encoded set of files in an
+HTTP POST, or streams in the raw data of a single file in an HTTP PUT.
+
+See file_uploader.py in this directory for code that uploads files in this format.
+"""
+
+import logging
+
+try:
+    from urllib.parse import unquote
+except ImportError:
+    # Python 2.
+    from urllib import unquote
+
+import tornado.ioloop
+import tornado.web
+from tornado import options
+
+
+class POSTHandler(tornado.web.RequestHandler):
+    def post(self):
+        for field_name, files in self.request.files.items():
+            for info in files:
+                filename, content_type = info['filename'], info['content_type']
+                body = info['body']
+                logging.info('POST "%s" "%s" %d bytes',
+                             filename, content_type, len(body))
+
+        self.write('OK')
+
+
+@tornado.web.stream_request_body
+class PUTHandler(tornado.web.RequestHandler):
+    def initialize(self):
+        self.bytes_read = 0
+
+    def data_received(self, chunk):
+        self.bytes_read += len(chunk)
+
+    def put(self, filename):
+        filename = unquote(filename)
+        mtype = self.request.headers.get('Content-Type')
+        logging.info('PUT "%s" "%s" %d bytes', filename, mtype, self.bytes_read)
+        self.write('OK')
+
+
+def make_app():
+    return tornado.web.Application([
+        (r"/post", POSTHandler),
+        (r"/(.*)", PUTHandler),
+    ])
+
+
+if __name__ == "__main__":
+    # Tornado configures logging.
+    options.parse_command_line()
+    app = make_app()
+    app.listen(8888)
+    tornado.ioloop.IOLoop.current().start()

demos/file_upload/file_uploader.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+
+"""Usage: python file_uploader.py [--put] file1.txt file2.png ...
+
+Demonstrates uploading files to a server, without concurrency. It can either
+POST a multipart-form-encoded request containing one or more files, or PUT a
+single file without encoding.
+
+See also file_receiver.py in this directory, a server that receives uploads.
+"""
+
+import mimetypes
+import os
+import sys
+from functools import partial
+from uuid import uuid4
+
+try:
+    from urllib.parse import quote
+except ImportError:
+    # Python 2.
+    from urllib import quote
+
+from tornado import gen, httpclient, ioloop
+from tornado.options import define, options
+
+
+# Using HTTP POST, upload one or more files in a single multipart-form-encoded
+# request.
+@gen.coroutine
+def multipart_producer(boundary, filenames, write):
+    boundary_bytes = boundary.encode()
+
+    for filename in filenames:
+        filename_bytes = filename.encode()
+        write(b'--%s\r\n' % (boundary_bytes,))
+        write(b'Content-Disposition: form-data; name="%s"; filename="%s"\r\n' %
+              (filename_bytes, filename_bytes))
+
+        mtype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
+        write(b'Content-Type: %s\r\n' % (mtype.encode(),))
+        write(b'\r\n')
+        with open(filename, 'rb') as f:
+            while True:
+                # 16k at a time.
+                chunk = f.read(16 * 1024)
+                if not chunk:
+                    break
+                write(chunk)
+
+                # Let the IOLoop process its event queue.
+                yield gen.moment
+
+        write(b'\r\n')
+        yield gen.moment
+
+    write(b'--%s--\r\n' % (boundary_bytes,))
+
+
+# Using HTTP PUT, upload one raw file. This is preferred for large files since
+# the server can stream the data instead of buffering it entirely in memory.
+@gen.coroutine
+def post(filenames):
+    client = httpclient.AsyncHTTPClient()
+    boundary = uuid4().hex
+    headers = {'Content-Type': 'multipart/form-data; boundary=%s' % boundary}
+    producer = partial(multipart_producer, boundary, filenames)
+    response = yield client.fetch('http://localhost:8888/post',
+                                  method='POST',
+                                  headers=headers,
+                                  body_producer=producer)
+
+    print(response)
+
+
+@gen.coroutine
+def raw_producer(filename, write):
+    with open(filename, 'rb') as f:
+        while True:
+            # 16K at a time.
+            chunk = f.read(16 * 1024)
+            if not chunk:
+                # Complete.
+                break
+
+            write(chunk)
+
+
+@gen.coroutine
+def put(filenames):
+    client = httpclient.AsyncHTTPClient()
+    for filename in filenames:
+        mtype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
+        headers = {'Content-Type': mtype}
+        producer = partial(raw_producer, filename)
+        url_path = quote(os.path.basename(filename))
+        response = yield client.fetch('http://localhost:8888/%s' % url_path,
+                                      method='PUT',
+                                      headers=headers,
+                                      body_producer=producer)
+
+        print(response)
+
+
+define("put", type=bool, help="Use PUT instead of POST", group="file uploader")
+
+# Tornado configures logging from command line opts and returns remaining args.
+filenames = options.parse_command_line()
+if not filenames:
+    print("Provide a list of filenames to upload.", file=sys.stderr)
+    sys.exit(1)
+
+method = put if options.put else post
+ioloop.IOLoop.current().run_sync(lambda: method(filenames))

docs/guide/structure.rst

@@ -153,6 +153,10 @@ By default uploaded files are fully buffered in memory; if you need to
 handle files that are too large to comfortably keep in memory see the
 `.stream_request_body` class decorator.
 
+In the demos directory,
+`file_receiver.py <https://github.com/tornadoweb/tornado/tree/master/demos/file_upload/>`_
+shows both methods of receiving file uploads.
+
 Due to the quirks of the HTML form encoding (e.g. the ambiguity around
 singular versus plural arguments), Tornado does not attempt to unify
 form arguments with other types of input.  In particular, we do not

docs/httpclient.rst

@@ -50,3 +50,11 @@ Implementations
 .. class:: CurlAsyncHTTPClient(io_loop, max_clients=10, defaults=None)
 
    ``libcurl``-based HTTP client.
+
+Example Code
+~~~~~~~~~~~~
+
+* `A simple webspider <https://github.com/tornadoweb/tornado/blob/master/demos/webspider/webspider.py>`_
+  shows how to fetch URLs concurrently.
+* `The file uploader demo <https://github.com/tornadoweb/tornado/tree/master/demos/file_upload/>`_
+  uses either HTTP POST or HTTP PUT to upload files to a server.

tornado/web.py

@@ -1670,6 +1670,9 @@ def stream_request_body(cls):
     There is a subtle interaction between ``data_received`` and asynchronous
     ``prepare``: The first call to ``data_received`` may occur at any point
     after the call to ``prepare`` has returned *or yielded*.
+
+    See the `file receiver demo <https://github.com/tornadoweb/tornado/tree/master/demos/file_upload/>`_
+    for example usage.
     """
     if not issubclass(cls, RequestHandler):
         raise TypeError("expected subclass of RequestHandler, got %r", cls)