How do I delay tcp traffic on a socket?

Question

I would like to test the interactions between a client program and a server program for race conditions. They connect to each other via tcp. The client does blocking calls to the server on multiple threads (one tcp connection to the server per thread). I would like to test the race condition where one blocking call finishes before the other.

To do this I was hoping to delay the tcp connections by different amounts (so I don't have to explicitly rewrite either the client or server).

I was hoping to do something like this:

socat tcp-listen:$pin system:"delaycat $delay |socat - 'tcp:localhost:$pout'"

where $pin is the port the client connects to, $delay is number of seconds to delay and $pout is the port the server listens on. And delaycat is an imaginary program which delays the input stream by n seconds.

Is there some existing program which does what I want? Or should I write delaycat?

Edit: note that a system wide delay would not really be ideal, as I'd like to delay the individual sockets by different amounts if possible.

Solution 1

I couldn't find anything already out there so I wrote a python script to do this (this is probably buggy):

#!/usr/bin/env python26
"""Add latency to a tcp connection"""
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from functools import partial

from twisted.internet.reactor import (
    listenTCP, connectTCP, callLater, run
)
from twisted.internet.protocol import (
    ServerFactory, ReconnectingClientFactory, Protocol
)


class ConnectionToServer(Protocol):
    def __init__(self, connection_from_client, delay_to_client):
        self._delay_to_client = delay_to_client
        self._connection_from_client = connection_from_client

    def connectionMade(self):
        self._connection_from_client.connection_to_server = self
        if self._connection_from_client.buffer_from_client:
            self.transport.write(
                self._connection_from_client.buffer_from_client
            )

    def dataReceived(self, data):
        callLater(
            self._delay_to_client,
            self._connection_from_client.transport.write, data
        )

    def connectionLost(self, reason):
        callLater(
            self._delay_to_client,
            self._connection_from_client.transport.loseConnection
        )


class ConnectionFromClient(Protocol):
    def __init__(self, server_host, server_port, delay_to_client, delay_to_server):
        self._delay_to_server = delay_to_server
        self.connection_to_server = None
        self.buffer_from_client = ''
        server_connection_factory = ReconnectingClientFactory()
        server_connection_factory.protocol = partial(
            ConnectionToServer, self, delay_to_client
        )
        self._server_connector = connectTCP(
            server_host, server_port, server_connection_factory
        )

    def dataReceived(self, data):
        callLater(self._delay_to_server, self._write, data)

    def connectionLost(self, reason):
        callLater(
            self._delay_to_server, self._server_connector.disconnect
        )

    def _write(self, data):
        if self.connection_to_server:
            self.connection_to_server.transport.write(data)
        else:
            self.buffer_from_client += data


def main():
    """Add latency to a tcp connection"""
    parser = ArgumentParser(
        description=main.__doc__,
        formatter_class=ArgumentDefaultsHelpFormatter
    )
    parser.add_argument(
        'client_port', type=int, help='client connects to this port'
    )
    parser.add_argument(
        'server_port', type=int, help='server listens on this port'
    )
    parser.add_argument(
        '-t', '--server-host', default='localhost',
        help='server is running on this host'
    )
    parser.add_argument(
        '-c', '--delay-to-client', default=0, type=float,
        help='messages to client are delayed by this many seconds'
    )
    parser.add_argument(
        '-s', '--delay-to-server', default=0, type=float,
        help='messages to server are delayed by this many seconds'
    )
    args = parser.parse_args()
    client_connection_factory = ServerFactory()
    client_connection_factory.protocol = partial(
        ConnectionFromClient, args.server_host, args.server_port,
        args.delay_to_client, args.delay_to_server
    )
    listenTCP(args.client_port, client_connection_factory)
    run()
if __name__ == '__main__':
    main()

Solution 2

On a linux machine you can use tc with Netem.

For instance, the command tc qdisc add dev eth0 root netem delay 100ms will delay all outgoing packets by 100 milliseconds. To delay incoming packets you can use the ifb - Intermediate Functional Block