How to use sendmsg() to send a file-descriptor via sockets between 2 processes?

Question

After @cnicutar answers me on this question, I tried to send a file-descriptor from the parent process to its child. Based on this example, I wrote this code:

int socket_fd ,accepted_socket_fd, on = 1;
int server_sd, worker_sd, pair_sd[2];
struct sockaddr_in client_address;
struct sockaddr_in server_address;

/* =======================================================================
 * Setup the network socket.
 * =======================================================================
 */

if((socket_fd = socket(AF_INET, SOCK_STREAM, 0)) < 0)
{
    perror("socket()");
    exit(EXIT_FAILURE);
}

if((setsockopt(socket_fd, SOL_SOCKET,  SO_REUSEADDR, (char *) &on, sizeof(on))) < 0)
{
    perror("setsockopt()");
    exit(EXIT_FAILURE);
}

server_address.sin_family = AF_INET;                 /* Internet address type */
server_address.sin_addr.s_addr = htonl(INADDR_ANY);  /* Set for any local IP */
server_address.sin_port = htons(port);               /* Set to the specified port */

if(bind(socket_fd, (struct sockaddr *) &server_address, sizeof(server_address)) < 0)
{
    perror("bind()");
    exit(EXIT_FAILURE);
}

if(listen(socket_fd, buffers) < 0)
{
    perror("listen()");
    exit(EXIT_FAILURE);
}

if(socketpair(AF_UNIX, SOCK_DGRAM, 0, pair_sd) < 0)
{
    socketpair("bind()");
    exit(EXIT_FAILURE);
}

server_sd = pair_sd[0];
worker_sd = pair_sd[1];



/* =======================================================================
 * Worker processes
 * =======================================================================
 */    

struct iovec   iov[1];
struct msghdr  child_msg;
char   msg_buffer[80];
int pass_sd, rc;


/* Here the parent process create a pool of worker processes (its children) */
for(i = 0; i < processes; i++)
{
    if(fork() == 0)
    {
        // ...

        /* Loop forever, serving the incoming request */
        for(;;)
        {

            memset(&child_msg,   0, sizeof(child_msg));
            memset(iov,    0, sizeof(iov));

            iov[0].iov_base = msg_buffer;
            iov[0].iov_len  = sizeof(msg_buffer);
            child_msg.msg_iov     = iov;
            child_msg.msg_iovlen  = 1;
            child_msg.msg_name    = (char *) &pass_sd;
            child_msg.msg_namelen = sizeof(pass_sd);

            printf("Waiting on recvmsg\n");
            rc = recvmsg(worker_sd, &child_msg, 0);
            if (rc < 0)
            {
               perror("recvmsg() failed");
               close(worker_sd);
               exit(-1);
            }
            else if (child_msg.msg_namelen <= 0)
            {
               printf("Descriptor was not received\n");
               close(worker_sd);
               exit(-1);
            }
            else
            {
               printf("Received descriptor = %d\n", pass_sd);
            }

            //.. Here the child process can handle the passed file descriptor
        }
    }

}



/* =======================================================================
 * The parent process
 * =======================================================================
 */

struct msghdr parent_msg;
size_t length;

/* Here the parent will accept the incoming requests and passed it to its children*/
for(;;)
{
    length = sizeof(client_address);
    if((accepted_socket_fd = accept(socket_fd, NULL, NULL)) < 0)
    {
        perror("accept()");
        exit(EXIT_FAILURE);
    }

    memset(&parent_msg, 0, sizeof(parent_msg));
    parent_msg.msg_name  = (char *) &accepted_socket_fd;
    parent_msg.msg_namelen = sizeof(accepted_socket_fd);

    if((sendmsg(server_sd, &parent_msg, 0)) < 0)
    {
        perror("sendmsg()");
        exit(EXIT_FAILURE);
    }

}

But unfortunately, I got this error:

sendmsg(): Invalid argument

What should I do to fix this problem? and am I using the msghdr structure correctly? because in the example I mentioned above, they use msg_accrights and msg_accrightslen and I got some error when I use them so I had to use msg_name and msg_namelen instead.

Solution 1

This is extremely hard to get right. I'd recommend just using a library that does it for you. One of the simplest is libancillary. It gives you two functions, one to send a file descriptor over a UNIX-domain socket and one to receive one. They are absurdly simple to use.

Solution 2

The problem is you are passing the file descriptor in a msg_name field. This is an address field, and it is not intended to pass arbitrary data.

In fact, the file descriptors should be passed in a special way so the kernel could duplicate the file descriptor for the receiving process (and maybe the descriptor will have another value after the duplicating). That's why there is a special ancillary message type (SCM_RIGHTS) to pass file descriptors.

The following would work (I omitted some of the error handling). In client:

memset(&child_msg,   0, sizeof(child_msg));
char cmsgbuf[CMSG_SPACE(sizeof(int))];
child_msg.msg_control = cmsgbuf; // make place for the ancillary message to be received
child_msg.msg_controllen = sizeof(cmsgbuf);

printf("Waiting on recvmsg\n");
rc = recvmsg(worker_sd, &child_msg, 0);
struct cmsghdr *cmsg = CMSG_FIRSTHDR(&child_msg);
if (cmsg == NULL || cmsg -> cmsg_type != SCM_RIGHTS) {
     printf("The first control structure contains no file descriptor.\n");
     exit(0);
}
memcpy(&pass_sd, CMSG_DATA(cmsg), sizeof(pass_sd));
printf("Received descriptor = %d\n", pass_sd);

In server:

memset(&parent_msg, 0, sizeof(parent_msg));
struct cmsghdr *cmsg;
char cmsgbuf[CMSG_SPACE(sizeof(accepted_socket_fd))];
parent_msg.msg_control = cmsgbuf;
parent_msg.msg_controllen = sizeof(cmsgbuf); // necessary for CMSG_FIRSTHDR to return the correct value
cmsg = CMSG_FIRSTHDR(&parent_msg);
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_RIGHTS;
cmsg->cmsg_len = CMSG_LEN(sizeof(accepted_socket_fd));
memcpy(CMSG_DATA(cmsg), &accepted_socket_fd, sizeof(accepted_socket_fd));
parent_msg.msg_controllen = cmsg->cmsg_len; // total size of all control blocks

if((sendmsg(server_sd, &parent_msg, 0)) < 0)
{
    perror("sendmsg()");
    exit(EXIT_FAILURE);
}

See also man 3 cmsg, there are some examples.

Comments

[Nakedible]

The user is not using AF_INET for socket passing, but an AF_UNIX socket created by socketpair.

[Jeff Learman]

After sendmsg, what's the state of the fd in the server? Is it open and needs to be closed, or does sendmsg/cmsg close it? In example code I never see server closing the fd, which would be necessary if sendmsg doesn't close it.

[fuzzyTew]

I believe it stays open and the OS closes it when the process terminates if you don't first. Haven't fully checked that though.