From 03bc12dd57591a796cfbbb1ea007a9b29dd3f1d5 Mon Sep 17 00:00:00 2001 From: nick Date: Thu, 14 Feb 2013 16:24:10 +0000 Subject: [PATCH] flexnbd read/write: Switch to a non-blocking connect() to allow us to time these out --- src/readwrite.c | 42 +++++++++-------- src/sockutil.c | 118 ++++++++++++++++++++++++++++++++++++++++++++++-- src/sockutil.h | 14 ++++-- 3 files changed, 145 insertions(+), 29 deletions(-) diff --git a/src/readwrite.c b/src/readwrite.c index 0f20f24..bb971ce 100644 --- a/src/readwrite.c +++ b/src/readwrite.c @@ -1,7 +1,8 @@ #include "nbdtypes.h" #include "ioutil.h" +#include "sockutil.h" #include "util.h" -#include "serve.h" +#include "serve.h" #include #include @@ -18,14 +19,14 @@ int socket_connect(struct sockaddr* to, struct sockaddr* from) if (NULL != from) { if ( 0 > bind(fd, from, sizeof(struct sockaddr_in6)) ){ warn( "bind() failed"); - close( fd ); + FATAL_IF_NEGATIVE( close( fd ), SHOW_ERRNO( "FIXME" ) ); return -1; } } - if ( 0 > connect(fd, to, sizeof(struct sockaddr_in6)) ) { - warn( "connect failed" ); - close( fd ); + if ( 0 > sock_try_connect( fd, to, sizeof( struct sockaddr_in6 ), 15 ) ) { + warn( SHOW_ERRNO( "connect failed" ) ); + FATAL_IF_NEGATIVE( close( fd ), SHOW_ERRNO( "FIXME" ) ); return -1; } @@ -88,7 +89,7 @@ void fill_request(struct nbd_request *request, int type, off64_t from, int len) void read_reply(int fd, struct nbd_request *request, struct nbd_reply *reply) { struct nbd_reply_raw reply_raw; - + ERROR_IF_NEGATIVE(readloop(fd, &reply_raw, sizeof(struct nbd_reply_raw)), "Couldn't read reply"); @@ -108,14 +109,15 @@ void read_reply(int fd, struct nbd_request *request, struct nbd_reply *reply) void wait_for_data( int fd, int timeout_secs ) { fd_set fds; - struct timeval tv = {timeout_secs, 0}; + struct timeval tv = { timeout_secs, 0 }; int selected; FD_ZERO( &fds ); FD_SET( fd, &fds ); - selected = select( FD_SETSIZE, - &fds, NULL, NULL, - timeout_secs >=0 ? &tv : NULL ); + + selected = sock_try_select( + FD_SETSIZE, &fds, NULL, NULL, timeout_secs >=0 ? &tv : NULL + ); FATAL_IF( -1 == selected, "Select failed" ); ERROR_IF( 0 == selected, "Timed out waiting for reply" ); @@ -126,16 +128,16 @@ void socket_nbd_read(int fd, off64_t from, int len, int out_fd, void* out_buf, i { struct nbd_request request; struct nbd_reply reply; - + fill_request(&request, REQUEST_READ, from, len); FATAL_IF_NEGATIVE(writeloop(fd, &request, sizeof(request)), "Couldn't write request"); wait_for_data( fd, timeout_secs ); read_reply(fd, &request, &reply); - + if (out_buf) { - FATAL_IF_NEGATIVE(readloop(fd, out_buf, len), + FATAL_IF_NEGATIVE(readloop(fd, out_buf, len), "Read failed"); } else { @@ -150,13 +152,13 @@ void socket_nbd_write(int fd, off64_t from, int len, int in_fd, void* in_buf, in { struct nbd_request request; struct nbd_reply reply; - + fill_request(&request, REQUEST_WRITE, from, len); ERROR_IF_NEGATIVE(writeloop(fd, &request, sizeof(request)), "Couldn't write request"); - + if (in_buf) { - ERROR_IF_NEGATIVE(writeloop(fd, in_buf, len), + ERROR_IF_NEGATIVE(writeloop(fd, in_buf, len), "Write failed"); } else { @@ -165,7 +167,7 @@ void socket_nbd_write(int fd, off64_t from, int len, int in_fd, void* in_buf, in "Splice failed" ); } - + wait_for_data( fd, timeout_secs ); read_reply(fd, &request, &reply); } @@ -200,13 +202,13 @@ int socket_nbd_disconnect( int fd ) fatal( error_type " connection failed." );\ }\ } - + void do_read(struct mode_readwrite_params* params) { params->client = socket_connect(¶ms->connect_to.generic, ¶ms->connect_from.generic); FATAL_IF_NEGATIVE( params->client, "Couldn't connect." ); CHECK_RANGE("read"); - socket_nbd_read(params->client, params->from, params->len, + socket_nbd_read(params->client, params->from, params->len, params->data_fd, NULL, 10); close(params->client); } @@ -216,7 +218,7 @@ void do_write(struct mode_readwrite_params* params) params->client = socket_connect(¶ms->connect_to.generic, ¶ms->connect_from.generic); FATAL_IF_NEGATIVE( params->client, "Couldn't connect." ); CHECK_RANGE("write"); - socket_nbd_write(params->client, params->from, params->len, + socket_nbd_write(params->client, params->from, params->len, params->data_fd, NULL, 10); close(params->client); } diff --git a/src/sockutil.c b/src/sockutil.c index 1adcbb5..bb9cdf8 100644 --- a/src/sockutil.c +++ b/src/sockutil.c @@ -1,7 +1,14 @@ +#include +#include + +#include +#include +#include + #include "sockutil.h" #include "util.h" -size_t sockaddr_size(const struct sockaddr* sa) +size_t sockaddr_size( const struct sockaddr* sa ) { size_t ret = 0; @@ -17,7 +24,7 @@ size_t sockaddr_size(const struct sockaddr* sa) return ret; } -const char* sockaddr_address_string(const struct sockaddr* sa, char* dest, size_t len) +const char* sockaddr_address_string( const struct sockaddr* sa, char* dest, size_t len ) { NULLCHECK( sa ); NULLCHECK( dest ); @@ -47,18 +54,35 @@ const char* sockaddr_address_string(const struct sockaddr* sa, char* dest, size_ return ret; } -int sock_set_reuseaddr(int fd, int optval) +int sock_set_reuseaddr( int fd, int optval ) { return setsockopt( fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval) ); } /* Set the tcp_nodelay option */ -int sock_set_tcp_nodelay(int fd, int optval) +int sock_set_tcp_nodelay( int fd, int optval ) { return setsockopt( fd, IPPROTO_TCP, TCP_NODELAY, &optval, sizeof(optval) ); } -int sock_try_bind(int fd, const struct sockaddr* sa) +int sock_set_nonblock( int fd, int optval ) +{ + int flags = fcntl( fd, F_GETFL ); + + if ( flags == -1 ) { + return -1; + } + + if ( optval ) { + flags = flags | O_NONBLOCK; + } else { + flags = flags & (~O_NONBLOCK); + } + + return fcntl( fd, F_SETFL, flags ); +} + +int sock_try_bind( int fd, const struct sockaddr* sa ) { int bind_result; char s_address[256]; @@ -102,3 +126,87 @@ int sock_try_bind(int fd, const struct sockaddr* sa) return bind_result; } +int sock_try_select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout) +{ + int result; + + do { + result = select(nfds, readfds, writefds, exceptfds, timeout); + if ( errno != EINTR ) { + break; + } + + } while ( result == -1 ); + + return result; +} + +int sock_try_connect( int fd, struct sockaddr* to, socklen_t addrlen, int wait ) +{ + fd_set fds; + struct timeval tv = { wait, 0 }; + int result = 0; + + if ( sock_set_nonblock( fd, 1 ) == -1 ) { + warn( SHOW_ERRNO( "Failed to set socket non-blocking for connect()" ) ); + return connect( fd, to, addrlen ); + } + + FD_ZERO( &fds ); + FD_SET( fd, &fds ); + + do { + result = connect( fd, to, addrlen ); + + if ( result == -1 ) { + switch( errno ) { + case EINPROGRESS: + result = 0; + break; /* success */ + case EAGAIN: + case EINTR: + break; /* Try again */ + default: + warn( SHOW_ERRNO( "Failed to connect()") ); + goto out; + } + } + } while ( result == -1 ); + + if ( -1 == sock_try_select( FD_SETSIZE, NULL, &fds, NULL, &tv) ) { + warn( SHOW_ERRNO( "failed to select() on non-blocking connect" ) ); + result = -1; + goto out; + } + + if ( !FD_ISSET( fd, &fds ) ) { + result = -1; + errno = ETIMEDOUT; + goto out; + } + + int scratch; + socklen_t s_size = sizeof( scratch ); + if ( getsockopt( fd, SOL_SOCKET, SO_ERROR, &scratch, &s_size ) == -1 ) { + result = -1; + warn( SHOW_ERRNO( "getsockopt() failed" ) ); + goto out; + } + + if ( scratch == EINPROGRESS ) { + scratch = ETIMEDOUT; + } + + result = scratch ? -1 : 0; + errno = scratch; + +out: + if ( sock_set_nonblock( fd, 0 ) == -1 ) { + warn( SHOW_ERRNO( "Failed to make socket blocking after connect()" ) ); + return -1; + } + + debug( "sock_try_connect: %i", result ); + return result; +} + diff --git a/src/sockutil.h b/src/sockutil.h index e60fc91..6cc860c 100644 --- a/src/sockutil.h +++ b/src/sockutil.h @@ -2,11 +2,9 @@ #define SOCKUTIL_H -#include +#include #include -#include -#include -#include +#include /* Returns the size of the sockaddr, or 0 on error */ size_t sockaddr_size(const struct sockaddr* sa); @@ -25,8 +23,16 @@ int sock_set_tcp_nodelay(int fd, int optval); /* TODO: Set the tcp_cork option */ // int sock_set_cork(int fd, int optval); +int sock_set_nonblock(int fd, int optval); + /* Attempt to bind the fd to the sockaddr, retrying common transient failures */ int sock_try_bind(int fd, const struct sockaddr* sa); +/* Try to call select(), retrying EINTR */ +int sock_try_select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout); + +/* Try to call connect(), timing out after wait seconds */ +int sock_try_connect( int fd, struct sockaddr* to, socklen_t addrlen, int wait ); + #endif