Winsock2 - How to use IOCP on client side

c++ windows sockets visual-c++ winsock2

14,053

Try something like this:

Client:

#include <iostream>
#include <string>
#include <winsock2.h> 
#pragma comment(lib, "ws2_32.lib") 

typedef struct 
{ 
    WSAOVERLAPPED Overlapped; 
    SOCKET Socket; 
    WSABUF wsaBuf; 
    char Buffer[1024];
    DWORD Flags;
} PER_IO_DATA, * LPPER_IO_DATA; 

static DWORD WINAPI ClientWorkerThread(LPVOID lpParameter) 
{ 
    HANDLE hCompletionPort = (HANDLE)lpParameter; 
    DWORD NumBytesRecv = 0; 
    ULONG CompletionKey; 
    LPPER_IO_DATA PerIoData; 

    while (GetQueuedCompletionStatus(hCompletionPort, &NumBytesRecv, &CompletionKey, (LPOVERLAPPED*)&PerIoData, INFINITE))
    {
        if (!PerIoData)
            continue;

        if (NumBytesRecv == 0) 
        {
            std::cout << "Server disconnected!\r\n\r\n";  
        }
        else
        {
            // use PerIoData->Buffer as needed...
            std::cout << std::string(PerIoData->Buffer, NumBytesRecv);

            PerIoData->wsaBuf.len = sizeof(PerIoData->Buffer); 
            PerIoData->Flags = 0; 

            if (WSARecv(PerIoData->Socket, &(PerIoData->wsaBuf), 1, &NumBytesRecv, &(PerIoData->Flags), &(PerIoData->Overlapped), NULL) == 0)
                continue;

            if (WSAGetLastError() == WSA_IO_PENDING)
                continue;
        }

        closesocket(PerIoData->Socket);
        delete PerIoData;
    } 

    return 0; 
} 

int main(void) 
{ 
    WSADATA WsaDat; 
    if (WSAStartup(MAKEWORD(2, 2), &WsaDat) != 0)
        return 0; 

    HANDLE hCompletionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0); 
    if (!hCompletionPort)
        return 0;

    SYSTEM_INFO systemInfo; 
    GetSystemInfo(&systemInfo); 

    for (DWORD i = 0; i < systemInfo.dwNumberOfProcessors; ++i) 
    { 
        HANDLE hThread = CreateThread(NULL, 0, ClientWorkerThread, hCompletionPort, 0, NULL); 
        CloseHandle(hThread); 
    } 

    SOCKET Socket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED); 
    if (Socket == INVALID_SOCKET)
        return 0;

    SOCKADDR_IN SockAddr; 
    SockAddr.sin_family = AF_INET; 
    SockAddr.sin_addr.s_addr = inet_addr("127.0.0.1"); 
    SockAddr.sin_port = htons(8888); 

    CreateIoCompletionPort((HANDLE)Socket, hCompletionPort, 0, 0); 

    if (WSAConnect(Socket, (SOCKADDR*)(&SockAddr), sizeof(SockAddr), NULL, NULL, NULL, NULL) == SOCKET_ERROR)
        return 0;

    PER_IO_DATA *pPerIoData = new PER_IO_DATA;
    ZeroMemory(pPerIoData, sizeof(PER_IO_DATA)); 

    pPerIoData->Socket = Socket; 
    pPerIoData->Overlapped.hEvent = WSACreateEvent(); 
    pPerIoData->wsaBuf.buf = pPerIoData->Buffer; 
    pPerIoData->wsaBuf.len = sizeof(pPerIoData->Buffer); 

    DWORD dwNumRecv;
    if (WSARecv(Socket, &(pPerIoData->wsaBuf), 1, &dwNumRecv, &(pPerIoData->Flags), &(pPerIoData->Overlapped), NULL) == SOCKET_ERROR)
    {
        if (WSAGetLastError() != WSA_IO_PENDING)
        {
            delete pPerIoData;
            return 0;
        }
    } 

    while (TRUE) 
        Sleep(1000); 

    shutdown(Socket, SD_BOTH); 
    closesocket(Socket); 

    WSACleanup(); 
    return 0; 
}

Server:

#include <iostream>  
#include <winsock2.h>  
#pragma comment(lib,"ws2_32.lib")  

typedef struct
{
    WSAOVERLAPPED Overlapped;
    SOCKET Socket;
    WSABUF wsaBuf;
    char Buffer[1024];
    DWORD BytesSent;
    DWORD BytesToSend;
} PER_IO_DATA, * LPPER_IO_DATA; 


static DWORD WINAPI ServerWorkerThread(LPVOID lpParameter)
{
    HANDLE hCompletionPort = (HANDLE)lpParameter;
    DWORD NumBytesSent = 0;
    ULONG CompletionKey;
    LPPER_IO_DATA PerIoData;

    while (GetQueuedCompletionStatus(hCompletionPort, &NumBytesSent, &CompletionKey, (LPOVERLAPPED*)&PerIoData, INFINITE))    
    {
        if (!PerIoData)
            continue;

        if (NumBytesSent == 0)
        {
            std::cout << "Client disconnected!\r\n\r\n";
        }
        else
        {
            PerIoData->BytesSent += NumBytesSent;
            if (PerIoData->BytesSent < PerIoData->BytesToSend)
            {
                PerIoData->wsaBuf.buf = &(PerIoData->Buffer[PerIoData->BytesSent]);
                PerIoData->wsaBuf.len = (PerIoData->BytesToSend - PerIoData->BytesSent);
            }
            else
            {
                PerIoData->wsaBuf.buf = PerIoData->Buffer;
                PerIoData->wsaBuf.len = strlen(PerIoData->Buffer);
                PerIoData->BytesSent = 0;
                PerIoData->BytesToSend = PerIoData->wsaBuf.len;
            }

            if (WSASend(PerIoData->Socket, &(PerIoData->wsaBuf), 1, &NumBytesSent, 0, &(PerIoData->Overlapped), NULL) == 0)
                continue;

            if (WSAGetLastError() == WSA_IO_PENDING)
                continue;
        }

        closesocket(PerIoData->Socket);
        delete PerIoData;
    }

    return 0;
} 

int main()  
{  
    WSADATA WsaDat;  
    if (WSAStartup(MAKEWORD(2,2), &WsaDat) != 0)
        return 0;  

    HANDLE hCompletionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
    if (!hCompletionPort)
        return 0;

    SYSTEM_INFO systemInfo;
    GetSystemInfo(&systemInfo);

    for (DWORD i = 0; i < systemInfo.dwNumberOfProcessors; ++i)
    {
        HANDLE hThread = CreateThread(NULL, 0, ServerWorkerThread, hCompletionPort, 0, NULL);
        CloseHandle(hThread);
    } 

    SOCKET listenSocket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);  
    if (listenSocket == INVALID_SOCKET)
        return 0;  

    SOCKADDR_IN server;
    ZeroMemory(&server, sizeof(server));
    server.sin_family = AF_INET;  
    server.sin_addr.s_addr = INADDR_ANY;  
    server.sin_port = htons(8888);  

    if (bind(listenSocket, (SOCKADDR*)(&server), sizeof(server)) != 0)
        return 0;  

    if (listen(listenSocket, 1) != 0)
        return 0;  

    std::cout << "Waiting for incoming connection...\r\n";  

    SOCKET acceptSocket;  
    do  
    {  
        sockaddr_in saClient;  
        int nClientSize = sizeof(saClient);  
        acceptSocket = WSAAccept(listenSocket, (SOCKADDR*)&saClient, &nClientSize, NULL, NULL);  
    }
    while (acceptSocket == INVALID_SOCKET);

    std::cout << "Client connected!\r\n\r\n";  

    CreateIoCompletionPort((HANDLE)acceptSocket, hCompletionPort, 0, 0); 

    LPPER_IO_DATA pPerIoData = new PER_IO_DATA;
    ZeroMemory(pPerIoData, sizeof(PER_IO_DATA));

    strcpy(pPerIoData->Buffer, "Welcome to the server!\r\n");  

    pPerIoData->Overlapped.hEvent = WSACreateEvent(); 
    pPerIoData->Socket = acceptSocket; 
    pPerIoData->wsaBuf.buf = pPerIoData->Buffer;  
    pPerIoData->wsaBuf.len = strlen(pPerIoData->Buffer);  
    pPerIoData->BytesToSend = pPerIoData->wsaBuf.len;  

    DWORD dwNumSent;
    if (WSASend(acceptSocket, &(pPerIoData->wsaBuf), 1, &dwNumSent, 0, &(pPerIoData->Overlapped), NULL) == SOCKET_ERROR)
    {
        if (WSAGetLastError() != WSA_IO_PENDING)
        {
            delete pPerIoData;
            return 0;
        }
    }  

    while (TRUE)  
        Sleep(1000);  

    shutdown(acceptSocket, SD_BOTH);  
    closesocket(acceptSocket);  

    WSACleanup();  
    return 0;  
}

14,053

Author by

jpen

Updated on June 18, 2022

Comments

jpen almost 2 years

I’ve recently started learning IOCP on Windows and been reading the following article:

http://www.codeproject.com/Tips/95363/Another-TCP-echo-server-using-IOCP

You can download the sample for the article from:

http://dl.dropbox.com/u/281215/documentation/iocp-1.00.html

The sample contains two simple applications – iocp_echo_server and TcpEchoClient.

I understand that IOCP is usually used on the server side of the client/server model but I’d like to create a client using IOCP.

I’ve so far tried modifying the client sample above so that whenever the server sends a response to the client, it gets picked up automatically, however it doesn’t work.

I’ve left iocp_echo_server.c as is. My modified version of TcpEchoClient.c looks like:

//TcpEchoClient.c - a minimalistic echo client
// -----------------------------------------------------------------------------

// C language includes
#include <stdio.h>
#include <winsock2.h>
#include "mswsock.h"  // for AcceptEx
#include <stdlib.h> // exit
#include <string.h>

// Windows includes
#include <windows.h>

#pragma warning(disable: 4996) // sprintf

// -----------------------------------------------------------------------------

// configuration
enum
{
    BUFLEN = 1000,
    SERVICE_PORT = 4000,
    SERVER_ADDRESS = INADDR_LOOPBACK
};

enum // socket operations
{
    OP_NONE,
    OP_ACCEPT,
    OP_READ,
    OP_WRITE
};

typedef struct _SocketState // socket state & control
{
    char operation;
    SOCKET socket;
    DWORD length;
    char buf[1024];
} SocketState;

// variables
static HANDLE cpl_port;

static SOCKET sock;
static SocketState sock_state;
static WSAOVERLAPPED sock_ovl;

static LPFN_ACCEPTEX pfAcceptEx;
static GUID GuidAcceptEx = WSAID_ACCEPTEX;

static int msgNumber;
static char msgBuf[BUFLEN];
static struct sockaddr_in sin;

// prototypes
static void createConnection(void);
static void createSocket(void);
static void init(void);
static void initWinsock(void);
static void prepareEndpoint(void);
static void recvBuffer(void);
static void run(void);
static void sendBuffer(void);

static SOCKET create_accepting_socket(void);
static void create_io_completion_port(void);
static BOOL get_completion_status(DWORD*, SocketState**,WSAOVERLAPPED**);

// -----------------------------------------------------------------------------

void main(void)
{
    init();
    run();
}

// -----------------------------------------------------------------------------

static void createConnection(void)
{
    printf("* connecting\n");
    if (WSAConnect(sock, (LPSOCKADDR)&sin, sizeof(sin), NULL, NULL, NULL, NULL) == SOCKET_ERROR)
    {
        int err = WSAGetLastError();
        printf("* error %d in connect\n", err);
        exit(1);
    }
    printf("* connected\n");
}

// -----------------------------------------------------------------------------

static void createSocket(void)
{
    sock = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
    if (sock == INVALID_SOCKET)
    {
        int err = WSAGetLastError();
        printf("* error %d creating socket\n", err);
        exit(1);
    }

    // for use by AcceptEx
    sock_state.socket = 0; // to be updated later
    sock_state.operation = OP_ACCEPT;

    if (CreateIoCompletionPort((HANDLE)sock, cpl_port, (ULONG_PTR)&sock_state, 0) != cpl_port)
    {
        int err = WSAGetLastError();
        printf("* error %d in listener\n", err);
        exit(1);
    }
}

// -----------------------------------------------------------------------------

static void init(void)
{
    initWinsock();
    create_io_completion_port();
    createSocket();
    prepareEndpoint();
    createConnection();
}

// -----------------------------------------------------------------------------

static void initWinsock(void)
{
    WSADATA wsaData;

    if (WSAStartup(0x202, &wsaData) == SOCKET_ERROR)
    {
        int err = WSAGetLastError();
        printf("* error %d in WSAStartup\n", err);
        exit(1);
    }
}
// -----------------------------------------------------------------------------

static void prepareEndpoint(void)
{
    sin.sin_family = AF_INET;
    sin.sin_addr.s_addr = htonl(SERVER_ADDRESS);
    sin.sin_port = htons(SERVICE_PORT);

    // bind_listening_socket()
    {
        //if (bind(sock, (SOCKADDR*)&sin, sizeof(sin)) == SOCKET_ERROR)
        //{
        //    printf("* error in bind!\n");
        //    exit(1);
        //}
    }

    // start_listening()
    {
        //if (listen(sock, 100) == SOCKET_ERROR)
        //{
        //    printf("* error in listen!\n");
        //    exit(1);
        //}
        //printf("* started listening for connection requests...\n");
    }

    // load_accept_ex()
    {
        //DWORD dwBytes;

        // black magic for me!!!
        // You do not need to call in your code WSAIoctl. You can directly use AcceptEx and adds Mswsock.lib.
        //WSAIoctl(sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &GuidAcceptEx, sizeof(GuidAcceptEx), &pfAcceptEx, sizeof(pfAcceptEx), &dwBytes, NULL, NULL);
    }

    // start_accepting()
    {
        //SOCKET acceptor = create_accepting_socket();
        //DWORD expected = sizeof(struct sockaddr_in) + 16;

        //printf("* started accepting connections...\n");

        // uses listener's completion key and overlapped structure
        //sock_state.socket = acceptor;
        //memset(&sock_ovl, 0, sizeof(WSAOVERLAPPED));

        // starts asynchronous accept
        //if (!pfAcceptEx(sock, acceptor, sock_state.buf, 0 /* no recv */, expected, expected, NULL, &sock_ovl))
        //{
        //    int err = WSAGetLastError();
        //    if (err != ERROR_IO_PENDING)
        //    {
        //        printf("* error %d in AcceptEx\n", err);
        //        exit(1);
        //    }
        //}
    }
}

// -----------------------------------------------------------------------------

static void recvBuffer(void)
{
    char* buf = msgBuf;
    int pendingLen = BUFLEN;

    printf("* receiving reply\n");

    while (pendingLen > 0)
    {
        int partialLen = recv(sock, buf, pendingLen, 0);

        if (partialLen > 0)
        {
            pendingLen -= partialLen;
            buf += partialLen;
            continue;
        }

        // ------

        if (partialLen == 0)
        {
            printf("* connection closed by the server\n");
        }
        else // partialLen < 0
        {
            int err = WSAGetLastError();
            printf("* error %d in recv\n", err);
        }

        exit(1);
    }
}

// -----------------------------------------------------------------------------

static void run(void)
{
    DWORD length;
    BOOL resultOk;
    WSAOVERLAPPED* ovl_res;
    SocketState* socketState;

    for (;;)
    {
        sendBuffer();

        resultOk = get_completion_status(&length, &socketState, &ovl_res);

        recvBuffer();
    }
}

// -----------------------------------------------------------------------------

static void sendBuffer(void)
{
    char* buf = msgBuf;
    int pendingLen = BUFLEN;

    printf("* sending message\n");
    sprintf(msgBuf, "%05 *****", msgNumber++);

    while (pendingLen > 0)
    {
        int partialLen = send(sock, buf, pendingLen, 0);

        if (partialLen > 0)
        {
            pendingLen -= partialLen;
            buf += partialLen;
            continue;
        }

        // -----------

        if (partialLen == 0)
        {
            printf("* connection closed by the server\n");
        }
        else // partialLen < 0
        {
            int err = WSAGetLastError();
            printf("* error %d in send\n", err);
        }

        exit(1);
    }
}

// -----------------------------------------------------------------------------

static SOCKET create_accepting_socket(void)
{
    SOCKET acceptor = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
    if (acceptor == INVALID_SOCKET)
    {
        printf("* error creating accept socket!\n");
        exit(1);
    }
    return acceptor;
}

// -----------------------------------------------------------------------------

static void create_io_completion_port(void)
{
    cpl_port = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
    if (!cpl_port)
    {
        int err = WSAGetLastError();
        printf("* error %d in line %d CreateIoCompletionPort\n", err, __LINE__);
        exit(1);
    }
}

// -----------------------------------------------------------------------------

static BOOL get_completion_status(DWORD* length, SocketState** socketState, WSAOVERLAPPED** ovl_res)
{
    BOOL resultOk;
    *ovl_res = NULL;
    *socketState = NULL;

    resultOk = GetQueuedCompletionStatus(cpl_port, length, (PULONG_PTR)socketState, ovl_res, INFINITE);

    if (!resultOk)
    {
        DWORD err = GetLastError();
        printf("* error %d getting completion port status!!!\n", err);
    }

    if (!*socketState || !*ovl_res)
    {
        printf("* don't know what to do, aborting!!!\n");
        exit(1);
    }

    return resultOk;
}

// -----------------------------------------------------------------------------
// the end

When the server send a response by calling:

WSASend(socketState->socket, &wsabuf, 1, NULL, 0, ovl, NULL)

I’d expect it to be picked up by the client on this line:

resultOk = get_completion_status(&length, &socketState, &ovl_res);

But it doesn’t…

Would anybody be able to tell me what I’m doing wrong?

Edit:

I’ve taken the following points:

On the client side, you use WSAConnect() to create an outbound connection.
Call WSARecv() and WSASend() to start reading/writing operations when needed
you have to use WSASend/WSARecv if you want to use I/O completion ports.

and attempted to create a simple IOCP based client:

#include <iostream>
#include <winsock2.h>
#pragma comment(lib,"ws2_32.lib")

static DWORD WINAPI ClientWorkerThread(LPVOID lpParameter);

int main(void)
{
    WSADATA WsaDat;
    if (WSAStartup(MAKEWORD(2, 2), &WsaDat) != NO_ERROR)
        return 0;

    // Step 1 - Create an I/O completion port.
    HANDLE hCompletionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
    if (!hCompletionPort)
        return 0;

    // Step 2 - Find how many processors.
    SYSTEM_INFO systemInfo;
    GetSystemInfo(&systemInfo);
    const int nNumberOfProcessors = systemInfo.dwNumberOfProcessors;

    // Step 3 - Create worker threads.
    for (int i = 0; i < nNumberOfProcessors; i++)
    {
        HANDLE hThread = CreateThread(NULL, 0, ClientWorkerThread, hCompletionPort, 0, NULL);
        CloseHandle(hThread);
    }

    // Step 4 - Create a socket.
    SOCKET Socket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
    if (Socket == INVALID_SOCKET)
        return 0;

    struct hostent *host;
    if ((host = gethostbyname("localhost")) == NULL)
        return 0;

    SOCKADDR_IN SockAddr;
    SockAddr.sin_family = AF_INET;
    SockAddr.sin_addr.s_addr = *((unsigned long*)host->h_addr);
    SockAddr.sin_port = htons(8888);

    // Step 5 - Associate the socket with the I/O completion port.
    CreateIoCompletionPort((HANDLE)Socket, hCompletionPort, (ULONG_PTR)0, 0);

    if (WSAConnect(Socket, (SOCKADDR*)(&SockAddr), sizeof(SockAddr), NULL, NULL, NULL, NULL) == SOCKET_ERROR)
        return 0;

    char buffer[1000];
    memset(buffer, 0, 999);
    WSABUF wsaBuf = {strlen(buffer), buffer};
    DWORD dwSendBytes = 0;
    DWORD dwReceivedBytes = 0;
    DWORD dwFlags = 0;
    WSAOVERLAPPED wsaOverlapped;
    SecureZeroMemory((PVOID)&wsaOverlapped, sizeof(wsaOverlapped));
    wsaOverlapped.hEvent = WSACreateEvent();

    for(;;)
    {
        WSARecv(Socket, &wsaBuf, 1, &dwReceivedBytes, &dwFlags, &wsaOverlapped, NULL);
        std::cout << wsaBuf.buf;

        //WSASend(Socket, &wsaBuf, 1, &dwSendBytes, 0, &wsaOverlapped, NULL);

        int nError = WSAGetLastError();
        if(nError != WSAEWOULDBLOCK&&nError != 0)
        {
            std::cout << "Winsock error code: " << nError << "\r\n";
            std::cout << "Server disconnected!\r\n";
            shutdown(Socket, SD_SEND);
            closesocket(Socket);

            break;
        }
        Sleep(1000);
    }

    WSACleanup();
    system("PAUSE");
    return 0;
}

static DWORD WINAPI ClientWorkerThread(LPVOID lpParameter)
{
    HANDLE hCompletionPort = (HANDLE)lpParameter;
    DWORD dwBytesTransferred = 0;

    while (TRUE)
    {
        BOOL bRet = GetQueuedCompletionStatus(hCompletionPort, &dwBytesTransferred, (LPDWORD)0, (LPOVERLAPPED*)0, INFINITE);
    }

    return 0;
}

I know there are several things I’m doing wrong but I don’t know what they are.

Could somebody take a look at my code and give me some hints please?

Many thanks

Edit 2:

Sorry this post is getting too long.

I've had another go trying to implement an IOCP based client after reading Remy's comments below but I'm still not sure if I'm on the right track.

I'd really appreciate it if somebody could take a look at my new code (compiles fine under VS2010 & error checking omitted) below and give me some feedback.

NonBlockingClient:

#include <iostream>
#include <winsock2.h>
#pragma comment(lib, "ws2_32.lib")

static DWORD WINAPI ClientWorkerThread(LPVOID lpParameter);

typedef struct _PER_HANDLE_DATA 
{
    SOCKET Socket;
} PER_HANDLE_DATA, * LPPER_HANDLE_DATA;

typedef struct
{
    WSAOVERLAPPED wsaOverlapped;
    WSABUF wsaBuf;
    int OperationType;
} PER_IO_DATA, * LPPER_IO_DATA;

int main(void)
{
    WSADATA WsaDat;
    WSAStartup(MAKEWORD(2, 2), &WsaDat);

    // Step 1 - Create an I/O completion port.
    HANDLE hCompletionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);

    // Step 2 - Find how many processors.
    SYSTEM_INFO systemInfo;
    GetSystemInfo(&systemInfo);

    // Step 3 - Create worker threads.
    for (int i = 0; i < (int)systemInfo.dwNumberOfProcessors; i++)
    {
        HANDLE hThread = CreateThread(NULL, 0, ClientWorkerThread, hCompletionPort, 0, NULL);
        CloseHandle(hThread);
    }

    // Step 4 - Create a socket.
    SOCKET Socket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);

    PER_HANDLE_DATA *pPerHandleData = new PER_HANDLE_DATA;
    pPerHandleData->Socket = Socket;

    struct hostent *host;
    host = gethostbyname("localhost");

    SOCKADDR_IN SockAddr;
    SockAddr.sin_family = AF_INET;
    SockAddr.sin_addr.s_addr = *((unsigned long*)host->h_addr);
    SockAddr.sin_port = htons(8888);

    // Step 5 - Associate the socket with the I/O completion port.
    CreateIoCompletionPort((HANDLE)Socket, hCompletionPort, (DWORD)pPerHandleData, 0);

    WSAConnect(Socket, (SOCKADDR*)(&SockAddr), sizeof(SockAddr), NULL, NULL, NULL, NULL);

    static char buffer[1000];
    memset(buffer, 0, 999);

    PER_IO_DATA *pPerIoData = new PER_IO_DATA;

    pPerIoData->wsaBuf.buf = buffer;
    pPerIoData->wsaBuf.len = sizeof(buffer);

    DWORD dwSendBytes = 0;
    DWORD dwReceivedBytes = 0;
    DWORD dwFlags = 0;

    SecureZeroMemory((PVOID)&pPerIoData->wsaOverlapped, sizeof(pPerIoData->wsaOverlapped));
    pPerIoData->wsaOverlapped.hEvent = WSACreateEvent();

    WSARecv(Socket, &pPerIoData->wsaBuf, 1, &dwReceivedBytes, &dwFlags, &pPerIoData->wsaOverlapped, NULL);
    std::cout << pPerIoData->wsaBuf.buf;

    for (;;)
    {
        int nError = WSAGetLastError();
        if (nError != WSAEWOULDBLOCK&&nError != 0)
        {
            std::cout << "Winsock error code: " << nError << "\r\n";
            std::cout << "Server disconnected!\r\n";
            shutdown(Socket, SD_SEND);
            closesocket(Socket);

            break;
        }

        Sleep(1000);
    }

    delete pPerHandleData;
    delete pPerIoData;
    WSACleanup();
    return 0;
}

static DWORD WINAPI ClientWorkerThread(LPVOID lpParameter)
{
    HANDLE hCompletionPort = (HANDLE)lpParameter;
    DWORD bytesCopied = 0;
    OVERLAPPED *overlapped = 0;
    LPPER_HANDLE_DATA PerHandleData;
    LPPER_IO_DATA PerIoData;
    DWORD SendBytes, RecvBytes;
    DWORD Flags;
    BOOL bRet;

    while (TRUE)
    {
        bRet = GetQueuedCompletionStatus(hCompletionPort, &bytesCopied, (LPDWORD)&PerHandleData, (LPOVERLAPPED*)&PerIoData, INFINITE);

        if (bytesCopied == 0)
        {
            break;
        }
        else
        {
            Flags = 0;
            ZeroMemory(&(PerIoData->wsaOverlapped), sizeof(WSAOVERLAPPED));

            PerIoData->wsaBuf.len = 1000;
            WSARecv(PerHandleData->Socket, &(PerIoData->wsaBuf), 1, &RecvBytes, &Flags, &(PerIoData->wsaOverlapped), NULL);
        }
    }

    return 0;
}

NonBlockingServer:

#include <iostream>
#include <winsock2.h>
#pragma comment(lib,"ws2_32.lib")

int main()
{
    WSADATA WsaDat;
    WSAStartup(MAKEWORD(2,2), &WsaDat);

    SOCKET listenSocket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);

    SOCKADDR_IN server;
    server.sin_family = AF_INET;
    server.sin_addr.s_addr = INADDR_ANY;
    server.sin_port = htons(8888);

    bind(listenSocket, (SOCKADDR*)(&server), sizeof(server));

    listen(listenSocket, 1);

    SOCKET acceptSocket = SOCKET_ERROR;
    sockaddr_in saClient;
    int nClientSize = sizeof(saClient);
    while (acceptSocket == SOCKET_ERROR)
    {
        std::cout << "Waiting for incoming connections...\r\n";
        acceptSocket = WSAAccept(listenSocket, (SOCKADDR*)&saClient, &nClientSize, NULL, NULL);
    }

    std::cout << "Client connected!\r\n\r\n";

    char *szMessage = "Welcome to the server!\r\n";
    WSAOVERLAPPED SendOverlapped;
    DWORD SendBytes;

    WSABUF DataBuf;
    DataBuf.len = 1000;
    DataBuf.buf = szMessage;

    SecureZeroMemory((PVOID)&SendOverlapped, sizeof(WSAOVERLAPPED));
    SendOverlapped.hEvent = WSACreateEvent();

    for (;;)
    {
        WSASend(acceptSocket, &DataBuf, 1, &SendBytes, 0, &SendOverlapped, NULL);

        int nError = WSAGetLastError();
        if (nError != WSAEWOULDBLOCK && nError != 0)
        {
            std::cout << "Winsock error code: " << nError << "\r\n";
            std::cout << "Client disconnected!\r\n";

            shutdown(acceptSocket, SD_SEND);
            closesocket(acceptSocket);

            break;
        }

        Sleep(1000);
    }

    WSACleanup();
    return 0;
}

Thanks again!

jpen almost 12 years

I'll take a look at the link you've provided. In iocp_echo_server.c, WSASend() and WSARecv() are used whereas in TcpEchoClient.c, send() and recv() are used. Is it a bad practice to mix blocking and non-blocking operations like this?
cmeerw almost 12 years

You just can't use I/O completion ports with blocking calls like recv and send - you have to use WSASend/WSARecv if you want to use I/O completion ports.
jpen almost 12 years

Please see my Edit 2 above. Cheers.
jpen almost 12 years

Thanks for your response. To compile the client code, I've done this cast: pPerIoData->wsaBuf.buf = (CHAR*)pPerIoData->Buffer;. WSARecv(Socket, &(pPerIoData->wsaBuf), 1, &(pPerIoData->BytesTransferred), NULL, &(pPerIoData->Overlapped), NULL) fails with this error: "0x0000271e The system detected an invalid pointer address in attempting to use a pointer argument in a call.". I haven't found a fix for it yet. Do you know how to fix this?
jpen almost 12 years

Also does an IOCP based client require an IOCP based server (my original server code doesn’t use IOCP but yours does)? I’m aiming to control a networked hardware device that acts as server. I send commands to it from my client application (currently not IOCP based) and the server sends responses back. But I don’t know how this server is implemented and don’t have any control over it.
Remy Lebeau almost 12 years

The WSABUF.buf type-cast is fine. 0x0000271e is WSAEFAULT and the docs say this about that error: "The lpBuffers parameter is not completely contained in a valid part of the user address space", which is BS, so I don't know what it is complaining about. An IOCP-based client does not require an IOCP-based server (and vice versa), just like a blocking client does not require a blocking server, and a non-blocking client does not require a non-blocking server.
jpen almost 12 years

It appears WSARecv() was failing because I was passing NULL for lpFlags. If I do DWORD dwFlags = 0; and pass &dwFlags instead, the function works fine.
Remy Lebeau almost 12 years

You would have to add it as a member of the PER_IO_DATA struct, not use it as a local variable, so that WSARecv() can write to it asynchronously at the same time it writes the received data to the buffer.
jpen almost 12 years

Could you explain what this line (pPerIoData->Overlapped.hEvent = WSACreateEvent()) is for in your code? Is this required because we’re passing NULL for “lpCompletionRoutine” when calling WSASend()/WSARecv()? Also when I run both programs (without stopping at breakpoints), “Welcome to the server!” is displayed 43 times and 44th one stops with “Welcome to the s”. The programs then hung. Any idea what’s going on?
Remy Lebeau almost 12 years

Yes, the hEvent is needed when the lpCompletionRoutine is NULL so the socket has something to signal when it is finished with an IOCP operation. GetQueuedCompletionStatus() then waits for that signal before returning the OVERLAPPED pointer and result values. As for the other issue, I made some tweaks to the code in my answer, see if it helps.
jpen almost 12 years

Hmm with the new code “Welcome to the server!” is displayed only once and the programs hung. By the way I’ve had to change the client side code to pass dwFlags to WSARecv() so the function will succeed.
Remy Lebeau almost 12 years

Sorry, I left out a couple of things in my last edit. Fixed now.
jpen almost 12 years

In your sample code, you put per-handle data (socket handle) and per-I/O operation data (anything worker thread needs to know when processing a completion packet) into one struct, rather than separating them into two. I suppose it doesn't matter doing this way but I wonder is there any reason for this? Thanks.
Remy Lebeau almost 12 years

The io handlers need to know which socket is being operated on so they can start new io operations on the same sockets. There is nothing really socket-specific in this example that needs to be tracked separately. In a real-world scenario, it might make more sense as program complexity grows. In this example, I just wanted to focus in showing how to start an io operation and wait for its completion before starting a new operation.