twisted get body of POST request

Question

Ok,

This should be simple, since people do it all the time. I want to get the body of a POST request sent a twisted Agent. This is created with a twisted FileBodyProducer. On the server side, I get a request object for my render_POST method.

How do I retrieve the body?

server:

from twisted.web import server, resource
from twisted.internet import reactor


class Simple(resource.Resource):
    isLeaf = True
    def render_GET(self, request):
        return "{0}".format(request.args.keys())
    def render_POST(self, request):
        return "{0}".format(request.data)
        with open(request.args['filename'][0], 'rb') as fd:
            fd.write(request.write())

site = server.Site(Simple())
reactor.listenTCP(8080, site)
reactor.run()

client:

from StringIO import StringIO

from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers

from twisted.web.client import FileBodyProducer
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol
from pprint import pformat

class BeginningPrinter(Protocol):
    def __init__(self, finished):
        self.finished = finished
        self.remaining = 1024 * 10

    def dataReceived(self, bytes):
        if self.remaining:
            display = bytes[:self.remaining]
            print 'Some data received:'
            print display
            self.remaining -= len(display)

    def connectionLost(self, reason):
        print 'Finished receiving body:', reason.getErrorMessage()
        self.finished.callback(None)

agent = Agent(reactor)
body = FileBodyProducer(StringIO("hello, world"))
d = agent.request(
    'POST',
    'http://127.0.0.1:8080/',
    Headers({'User-Agent': ['Twisted Web Client Example'],
             'Content-Type': ['text/x-greeting']}),
    body)

def cbRequest(response):
    print 'Response version:', response.version
    print 'Response code:', response.code
    print 'Response phrase:', response.phrase
    print 'Response headers:'
    print pformat(list(response.headers.getAllRawHeaders()))
    finished = Deferred()
    response.deliverBody(BeginningPrinter(finished))
    return finished
d.addCallback(cbRequest)

def cbShutdown(ignored):
    reactor.stop()
d.addBoth(cbShutdown)

reactor.run()

The only docs I can find for setting up the consumer side leave something to be desired. Primarily, how can a consumer use the write(data) method to receive results?

Which bit am I missing to plug these two components together?

Solution 1

All right, so it's as simple as calling request.content.read(). This, as far as I can tell, is undocumented in the API.

Here's the updated code for the client:

from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers

from twisted.web.client import FileBodyProducer
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol
from pprint import pformat

class BeginningPrinter(Protocol):
    def __init__(self, finished):
        self.finished = finished
        self.remaining = 1024 * 10

    def dataReceived(self, bytes):
        if self.remaining:
            display = bytes[:self.remaining]
            print 'Some data received:'
            print display
            self.remaining -= len(display)

    def connectionLost(self, reason):
        print 'Finished receiving body:', reason.getErrorMessage()
        self.finished.callback(None)

class SaveContents(Protocol):
    def __init__(self, finished, filesize, filename):
        self.finished = finished
        self.remaining = filesize
        self.outfile = open(filename, 'wb')

    def dataReceived(self, bytes):
        if self.remaining:
            display = bytes[:self.remaining]
            self.outfile.write(display)
            self.remaining -= len(display)
        else:
            self.outfile.close()

    def connectionLost(self, reason):
        print 'Finished receiving body:', reason.getErrorMessage()
        self.outfile.close()
        self.finished.callback(None)

agent = Agent(reactor)
f = open('70935-new_barcode.pdf', 'rb')
body = FileBodyProducer(f)
d = agent.request(
    'POST',
    'http://127.0.0.1:8080?filename=test.pdf',
    Headers({'User-Agent': ['Twisted Web Client Example'],
             'Content-Type': ['multipart/form-data; boundary=1024'.format()]}),
    body)

def cbRequest(response):
    print 'Response version:', response.version
    print 'Response code:', response.code
    print 'Response phrase:', response.phrase
    print 'Response headers:'
    print 'Response length:', response.length
    print pformat(list(response.headers.getAllRawHeaders()))
    finished = Deferred()
    response.deliverBody(SaveContents(finished, response.length, 'test2.pdf'))
    return finished
d.addCallback(cbRequest)

def cbShutdown(ignored):
    reactor.stop()
d.addBoth(cbShutdown)

reactor.run()

And here's the server:

from twisted.web import server, resource
from twisted.internet import reactor
import os

# multi part encoding example: http://marianoiglesias.com.ar/python/file-uploading-with-multi-part-encoding-using-twisted/
class Simple(resource.Resource):
    isLeaf = True
    def render_GET(self, request):
        return "{0}".format(request.args.keys())
    def render_POST(self, request):
        with open(request.args['filename'][0], 'wb') as fd:
            fd.write(request.content.read())
        request.setHeader('Content-Length', os.stat(request.args['filename'][0]).st_size)
        with open(request.args['filename'][0], 'rb') as fd:
            request.write(fd.read())
        request.finish()
        return server.NOT_DONE_YET

site = server.Site(Simple())
reactor.listenTCP(8080, site)
reactor.run()

I can now write the file contents I receive, and read back the results.

Solution 2

If the content type is application/x-www-form-urlencoded or multipart/form-data, the body will be parsed and put in the request.args dict.

If the body is too big, it is written in temp file, otherwise in StringIO.

After the body is read, the method finish() is called. You can subclass Request and pares the body in this method or do sth else.

Comments

[Chris.Wilson]

For future reference, I would recommend not doing the request.write(fd.read()) with content-length set by os.stat().st_size. I had an issue where I was getting an IOError no space left on device at the request.write(fd.read()). If you have the memory, I'd recommend reading the file into a buffer first, and then doing content-length based on len(buf). then request.write(buf).