flexnbd read/write: Switch to a non-blocking connect() to allow us to time these out

src/readwrite.c

@@ -1,7 +1,8 @@
 #include "nbdtypes.h"
 #include "ioutil.h"
+#include "sockutil.h"
 #include "util.h"
-#include "serve.h" 
+#include "serve.h"
 
 #include <stdlib.h>
 #include <string.h>
@@ -18,14 +19,14 @@ int socket_connect(struct sockaddr* to, struct sockaddr* from)
 	if (NULL != from) {
 		if ( 0 > bind(fd, from, sizeof(struct sockaddr_in6)) ){
 			warn( "bind() failed");
-			close( fd );
+			FATAL_IF_NEGATIVE( close( fd ), SHOW_ERRNO( "FIXME" ) );
 			return -1;
 		}
 	}
 
-	if ( 0 > connect(fd, to, sizeof(struct sockaddr_in6)) ) {
+	if ( 0 > sock_try_connect( fd, to, sizeof( struct sockaddr_in6 ), 15 ) ) {
-		warn( "connect failed" );
+		warn( SHOW_ERRNO( "connect failed" ) );
-		close( fd );
+		FATAL_IF_NEGATIVE( close( fd ), SHOW_ERRNO( "FIXME" ) );
 		return -1;
 	}
 
@@ -88,7 +89,7 @@ void fill_request(struct nbd_request *request, int type, off64_t from, int len)
 void read_reply(int fd, struct nbd_request *request, struct nbd_reply *reply)
 {
 	struct nbd_reply_raw reply_raw;
-		
+
 	ERROR_IF_NEGATIVE(readloop(fd, &reply_raw, sizeof(struct nbd_reply_raw)),
 	  "Couldn't read reply");
 
@@ -108,14 +109,15 @@ void read_reply(int fd, struct nbd_request *request, struct nbd_reply *reply)
 void wait_for_data( int fd, int timeout_secs )
 {
 	fd_set fds;
-	struct timeval tv = {timeout_secs, 0};
+	struct timeval tv = { timeout_secs, 0 };
 	int selected;
 
 	FD_ZERO( &fds );
 	FD_SET( fd, &fds );
-	selected = select( FD_SETSIZE, 
+
-			&fds, NULL, NULL, 
+	selected = sock_try_select(
-			timeout_secs >=0 ? &tv : NULL );
+		FD_SETSIZE, &fds, NULL, NULL, timeout_secs >=0 ? &tv : NULL
+	);
 
 	FATAL_IF( -1 == selected, "Select failed" );
 	ERROR_IF(  0 == selected, "Timed out waiting for reply" );
@@ -126,16 +128,16 @@ void socket_nbd_read(int fd, off64_t from, int len, int out_fd, void* out_buf, i
 {
 	struct nbd_request request;
 	struct nbd_reply   reply;
-	
+
 	fill_request(&request, REQUEST_READ, from, len);
 	FATAL_IF_NEGATIVE(writeloop(fd, &request, sizeof(request)),
 	  "Couldn't write request");
 
 	wait_for_data( fd, timeout_secs );
 	read_reply(fd, &request, &reply);
-	
+
 	if (out_buf) {
-		FATAL_IF_NEGATIVE(readloop(fd, out_buf, len), 
+		FATAL_IF_NEGATIVE(readloop(fd, out_buf, len),
 		  "Read failed");
 	}
 	else {
@@ -150,13 +152,13 @@ void socket_nbd_write(int fd, off64_t from, int len, int in_fd, void* in_buf, in
 {
 	struct nbd_request request;
 	struct nbd_reply   reply;
-	
+
 	fill_request(&request, REQUEST_WRITE, from, len);
 	ERROR_IF_NEGATIVE(writeloop(fd, &request, sizeof(request)),
 	  "Couldn't write request");
-	
+
 	if (in_buf) {
-		ERROR_IF_NEGATIVE(writeloop(fd, in_buf, len), 
+		ERROR_IF_NEGATIVE(writeloop(fd, in_buf, len),
 		  "Write failed");
 	}
 	else {
@@ -165,7 +167,7 @@ void socket_nbd_write(int fd, off64_t from, int len, int in_fd, void* in_buf, in
 			"Splice failed"
 		);
 	}
-	
+
 	wait_for_data( fd, timeout_secs );
 	read_reply(fd, &request, &reply);
 }
@@ -200,13 +202,13 @@ int socket_nbd_disconnect( int fd )
 		fatal( error_type " connection failed." );\
 	}\
 }
-  
+
 void do_read(struct mode_readwrite_params* params)
 {
 	params->client = socket_connect(&params->connect_to.generic, &params->connect_from.generic);
 	FATAL_IF_NEGATIVE( params->client, "Couldn't connect." );
 	CHECK_RANGE("read");
-	socket_nbd_read(params->client, params->from, params->len, 
+	socket_nbd_read(params->client, params->from, params->len,
 	  params->data_fd, NULL, 10);
 	close(params->client);
 }
@@ -216,7 +218,7 @@ void do_write(struct mode_readwrite_params* params)
 	params->client = socket_connect(&params->connect_to.generic, &params->connect_from.generic);
 	FATAL_IF_NEGATIVE( params->client, "Couldn't connect." );
 	CHECK_RANGE("write");
-	socket_nbd_write(params->client, params->from, params->len, 
+	socket_nbd_write(params->client, params->from, params->len,
 	  params->data_fd, NULL, 10);
 	close(params->client);
 }

src/sockutil.c

@@ -1,7 +1,14 @@
+#include <unistd.h>
+#include <fcntl.h>
+
+#include <sys/types.h>
+#include <arpa/inet.h>
+#include <netinet/tcp.h>
+
 #include "sockutil.h"
 #include "util.h"
 
-size_t sockaddr_size(const struct sockaddr* sa)
+size_t sockaddr_size( const struct sockaddr* sa )
 {
 	size_t ret = 0;
 
@@ -17,7 +24,7 @@ size_t sockaddr_size(const struct sockaddr* sa)
 	return ret;
 }
 
-const char* sockaddr_address_string(const struct sockaddr* sa, char* dest, size_t len)
+const char* sockaddr_address_string( const struct sockaddr* sa, char* dest, size_t len )
 {
 	NULLCHECK( sa );
 	NULLCHECK( dest );
@@ -47,18 +54,35 @@ const char* sockaddr_address_string(const struct sockaddr* sa, char* dest, size_
 	return ret;
 }
 
-int sock_set_reuseaddr(int fd, int optval)
+int sock_set_reuseaddr( int fd, int optval )
 {
 	return setsockopt( fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval) );
 }
 
 /* Set the tcp_nodelay option */
-int sock_set_tcp_nodelay(int fd, int optval)
+int sock_set_tcp_nodelay( int fd, int optval )
 {
 	return setsockopt( fd, IPPROTO_TCP, TCP_NODELAY, &optval, sizeof(optval) );
 }
 
-int sock_try_bind(int fd, const struct sockaddr* sa)
+int sock_set_nonblock( int fd, int optval )
+{
+	int flags = fcntl( fd, F_GETFL );
+
+	if ( flags == -1 ) {
+		return -1;
+	}
+
+	if ( optval ) {
+		flags = flags | O_NONBLOCK;
+	} else {
+		flags = flags & (~O_NONBLOCK);
+	}
+
+	return fcntl( fd, F_SETFL, flags );
+}
+
+int sock_try_bind( int fd, const struct sockaddr* sa )
 {
 	int bind_result;
 	char s_address[256];
@@ -102,3 +126,87 @@ int sock_try_bind(int fd, const struct sockaddr* sa)
 	return bind_result;
 }
 
+int sock_try_select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout)
+{
+	int result;
+
+	do {
+		result = select(nfds, readfds, writefds, exceptfds, timeout);
+		if ( errno != EINTR ) {
+			break;
+		}
+
+	} while ( result == -1 );
+
+	return result;
+}
+
+int sock_try_connect( int fd, struct sockaddr* to, socklen_t addrlen, int wait )
+{
+	fd_set fds;
+	struct timeval tv = { wait, 0 };
+	int result = 0;
+
+	if ( sock_set_nonblock( fd, 1 ) == -1 ) {
+		warn( SHOW_ERRNO( "Failed to set socket non-blocking for connect()" ) );
+		return connect( fd, to, addrlen );
+	}
+
+	FD_ZERO( &fds );
+	FD_SET( fd, &fds );
+
+	do {
+		result = connect( fd, to, addrlen );
+
+		if ( result == -1 ) {
+			switch( errno ) {
+				case EINPROGRESS:
+					result = 0;
+					break; /* success */
+				case EAGAIN:
+				case EINTR:
+					break; /* Try again */
+				default:
+					warn( SHOW_ERRNO( "Failed to connect()") );
+					goto out;
+			}
+		}
+	} while ( result == -1 );
+
+	if ( -1 == sock_try_select( FD_SETSIZE, NULL, &fds, NULL, &tv) ) {
+		warn( SHOW_ERRNO( "failed to select() on non-blocking connect" ) );
+		result = -1;
+		goto out;
+	}
+
+	if ( !FD_ISSET( fd, &fds ) ) {
+		result = -1;
+		errno = ETIMEDOUT;
+		goto out;
+	}
+
+	int scratch;
+	socklen_t s_size = sizeof( scratch );
+	if ( getsockopt( fd, SOL_SOCKET, SO_ERROR, &scratch, &s_size ) == -1 ) {
+		result = -1;
+		warn( SHOW_ERRNO( "getsockopt() failed" ) );
+		goto out;
+	}
+
+	if ( scratch == EINPROGRESS ) {
+		scratch = ETIMEDOUT;
+	}
+
+	result = scratch ? -1 : 0;
+	errno = scratch;
+
+out:
+	if ( sock_set_nonblock( fd, 0 ) == -1 ) {
+		warn( SHOW_ERRNO( "Failed to make socket blocking after connect()" ) );
+		return -1;
+	}
+
+	debug( "sock_try_connect: %i", result );
+	return result;
+}
+

src/sockutil.h

@@ -2,11 +2,9 @@
 
 #define SOCKUTIL_H
 
-#include <sys/types.h>
+#include <sys/time.h>
 #include <sys/socket.h>
-#include <sys/un.h>
+#include <sys/select.h>
-#include <arpa/inet.h>
-#include <netinet/tcp.h>
 
 /* Returns the size of the sockaddr, or 0 on error */
 size_t sockaddr_size(const struct sockaddr* sa);
@@ -25,8 +23,16 @@ int sock_set_tcp_nodelay(int fd, int optval);
 /* TODO: Set the tcp_cork option */
 // int sock_set_cork(int fd, int optval);
 
+int sock_set_nonblock(int fd, int optval);
+
 /* Attempt to bind the fd to the sockaddr, retrying common transient failures */
 int sock_try_bind(int fd, const struct sockaddr* sa);
 
+/* Try to call select(), retrying EINTR */
+int sock_try_select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout);
+
+/* Try to call connect(), timing out after wait seconds */
+int sock_try_connect( int fd, struct sockaddr* to, socklen_t addrlen, int wait );
+
 #endif