thread polling for timestamp on socket's error queue doesn't wake up

Nov. 10, 2017

      I've notice that polling for timestamp in socket's error queue from a thread
that hasn't sended the packet results in not waking the polling thread when an
event arrives.

A simple proof of that is the following code. When compiled with
SEND_IN_SAME_THREAD defined, the program will poll on the same thread it has
sent the packet. Otherwise the waiter thread polls and the main thread send
packets.

Please first look in the two SEND_IN_SAME_THREAD preprocessor conditionals
before check the rest of the code.

Is this the expected behavior? Should I block waiting for packet to be sent
while I could send more packets?

kernel documentation about network timestamping (see section 1.3):
https://www.kernel.org/doc/Documentation/networking/timestamping.txt

kernel version: 4.12.14
glibc version: 2.24

/*
 * 10/11/2017
 * polling for timestamp on socket's error queue
 */

#include <arpa/inet.h> /* hton*() */
#include <poll.h> /* poll() */
#include <pthread.h> /* pthread_*() */
#include <string.h> /* memset() */
#include <sys/types.h> /* socket() recv() */
#include <sys/socket.h> /* socket() recv() */
#include <unistd.h> /* close() */

#include <linux/net_tstamp.h> /* timestamp stuff */

#define PORT  8080

static int
do_poll(int sfd)
{
	struct pollfd pfd;

	/* prepare */
	memset(&pfd, 0, sizeof(pfd));
	pfd.fd = sfd;
	pfd.events = POLLERR;

	/* poll */
	return poll(&pfd, 1, -1);
}

static void
do_recv(int sfd)
{
	char tmp;

	while (recv(sfd, &tmp, sizeof(tmp), MSG_ERRQUEUE) == sizeof(tmp));
}

static void
do_send(int sfd)
{
	struct sockaddr_in saddr;
	char tmp = 'a';

	/* prepare */
	saddr.sin_family = AF_INET;
	saddr.sin_port = htons(PORT);
	saddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);

	sendto(sfd, &tmp, sizeof(tmp), 0, (struct sockaddr*) &saddr,
	       sizeof(saddr));

	sleep(1);
}

static void*
waiter(void *data)
{
	int sfd = *((int*) data);

	for (;;) {
#ifdef SEND_IN_SAME_THREAD
		do_send(sfd);
#endif
		do_poll(sfd);
		do_recv(sfd);
	}

	return NULL;
}

static int
set_ts_opt(int sfd)
{
	unsigned int opt;

	/* set timestamp option */
	opt = SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_TX_SCHED |
	      SOF_TIMESTAMPING_OPT_CMSG;
	if (setsockopt(sfd, SOL_SOCKET, SO_TIMESTAMPING, (char*) &opt,
	               sizeof(opt)) == -1)
		return -1;

	return 0;
}

static int
open_socket(void)
{
	int sfd;
	int tmp;

	/* open socket in non-blocking mode */
	sfd = socket(AF_INET, SOCK_DGRAM|SOCK_NONBLOCK, 0);
	if (sfd == -1)
		return -1;

	tmp = set_ts_opt(sfd);
	if (tmp == -1)
		goto _go_close_socket;

	return sfd;

_go_close_socket:
	close(sfd);
	return -1;
}

int
main(int argc, char **argv)
{
	pthread_t waiter_thread;
	int sfd;

	if ((sfd = open_socket()) == -1)
		return 1;

	/* create thread and wait for it to terminate */
	pthread_create(&waiter_thread, NULL, waiter, &sfd);
#ifndef SEND_IN_SAME_THREAD
	for (;;)
		do_send(sfd);
#endif
	pthread_join(waiter_thread, NULL);

	return 0;
}

Cheers!

-- 
Ricardo Biehl Pasquali