diff --git a/src/client.c b/src/client.c index 092689f..ad68f6e 100644 --- a/src/client.c +++ b/src/client.c @@ -171,8 +171,9 @@ int client_read_request( struct client * client , struct nbd_request *out_reques CLIENT_ERROR_ON_FAILURE(select(FD_SETSIZE, &fds, NULL, NULL, NULL), "select() failed"); - if ( self_pipe_fd_isset( client->stop_signal, &fds ) ) + if ( self_pipe_fd_isset( client->stop_signal, &fds ) ){ return 0; + } if (readloop(client->socket, &request_raw, sizeof(request_raw)) == -1) { if (errno == 0) { diff --git a/src/client.h b/src/client.h index 90a6bee..eccdcc1 100644 --- a/src/client.h +++ b/src/client.h @@ -1,6 +1,7 @@ #ifndef CLIENT_H #define CLIENT_H + struct client { int socket; diff --git a/src/flexnbd.c b/src/flexnbd.c index b334594..d6e41e7 100644 --- a/src/flexnbd.c +++ b/src/flexnbd.c @@ -45,48 +45,6 @@ void exit_err( char *msg ) exit( 1 ); } -void params_serve( - struct server* out, - char* s_ip_address, - char* s_port, - char* s_file, - char *s_ctrl_sock, - int default_deny, - int acl_entries, - char** s_acl_entries ) -{ - out->tcp_backlog = 10; /* does this need to be settable? */ - - if (s_ip_address == NULL) - SERVER_ERROR("No IP address supplied"); - if (s_port == NULL) - SERVER_ERROR("No port number supplied"); - if (s_file == NULL) - SERVER_ERROR("No filename supplied"); - - if (parse_ip_to_sockaddr(&out->bind_to.generic, s_ip_address) == 0) - SERVER_ERROR("Couldn't parse server address '%s' (use 0 if " - "you want to bind to all IPs)", s_ip_address); - - /* control_socket_name is optional. It just won't get created if - * we pass NULL. */ - out->control_socket_name = s_ctrl_sock; - - out->acl = acl_create( acl_entries, s_acl_entries, default_deny ); - if (out->acl && out->acl->len != acl_entries) - SERVER_ERROR("Bad ACL entry '%s'", s_acl_entries[out->acl->len]); - - out->bind_to.v4.sin_port = atoi(s_port); - if (out->bind_to.v4.sin_port < 0 || out->bind_to.v4.sin_port > 65535) - SERVER_ERROR("Port number must be >= 0 and <= 65535"); - out->bind_to.v4.sin_port = htobe16(out->bind_to.v4.sin_port); - - out->filename = s_file; - out->filename_incomplete = xmalloc(strlen(s_file)+11+1); - strcpy(out->filename_incomplete, s_file); - strcpy(out->filename_incomplete + strlen(s_file), ".INCOMPLETE"); -} - /* TODO: Separate this function. * It should be: * params_read( struct mode_readwrite_params* out, @@ -295,7 +253,7 @@ int mode_serve( int argc, char *argv[] ) int default_deny = 0; // not on by default int err = 0; - struct server serve; + struct server * serve; while (1) { c = getopt_long(argc, argv, serve_short_options, serve_options, NULL); @@ -315,9 +273,9 @@ int mode_serve( int argc, char *argv[] ) } if ( err ) { exit_err( serve_help_text ); } - memset( &serve, 0, sizeof( serve ) ); - params_serve( &serve, ip_addr, ip_port, file, sock, default_deny, argc - optind, argv + optind ); - do_serve( &serve ); + serve = server_create( ip_addr, ip_port, file, sock, default_deny, argc - optind, argv + optind ); + do_serve( serve ); + server_destroy( serve ); return 0; } diff --git a/src/serve.c b/src/serve.c index 96a74ff..83f19c5 100644 --- a/src/serve.c +++ b/src/serve.c @@ -34,6 +34,75 @@ static inline void* sockaddr_address_data(struct sockaddr* sockaddr) return NULL; } +struct server * server_create ( + char* s_ip_address, + char* s_port, + char* s_file, + char *s_ctrl_sock, + int default_deny, + int acl_entries, + char** s_acl_entries ) +{ + struct server * out; + out = xmalloc( sizeof( struct server ) ); + + out->tcp_backlog = 10; /* does this need to be settable? */ + + if (s_ip_address == NULL) + SERVER_ERROR("No IP address supplied"); + if (s_port == NULL) + SERVER_ERROR("No port number supplied"); + if (s_file == NULL) + SERVER_ERROR("No filename supplied"); + + if (parse_ip_to_sockaddr(&out->bind_to.generic, s_ip_address) == 0) + SERVER_ERROR("Couldn't parse server address '%s' (use 0 if " + "you want to bind to all IPs)", s_ip_address); + + /* control_socket_name is optional. It just won't get created if + * we pass NULL. */ + out->control_socket_name = s_ctrl_sock; + + out->acl = acl_create( acl_entries, s_acl_entries, default_deny ); + if (out->acl && out->acl->len != acl_entries) + SERVER_ERROR("Bad ACL entry '%s'", s_acl_entries[out->acl->len]); + + out->bind_to.v4.sin_port = atoi(s_port); + if (out->bind_to.v4.sin_port < 0 || out->bind_to.v4.sin_port > 65535) + SERVER_ERROR("Port number must be >= 0 and <= 65535"); + out->bind_to.v4.sin_port = htobe16(out->bind_to.v4.sin_port); + + out->filename = s_file; + out->filename_incomplete = xmalloc(strlen(s_file)+11+1); + strcpy(out->filename_incomplete, s_file); + strcpy(out->filename_incomplete + strlen(s_file), ".INCOMPLETE"); + + pthread_mutex_init(&out->l_io, NULL); + pthread_mutex_init(&out->l_acl, NULL); + + out->close_signal = self_pipe_create(); + out->acl_updated_signal = self_pipe_create(); + + NULLCHECK( out->close_signal ); + NULLCHECK( out->acl_updated_signal ); + + return out; +} + +void server_destroy( struct server * serve ) +{ + self_pipe_destroy( serve->acl_updated_signal ); + self_pipe_destroy( serve->close_signal ); + + pthread_mutex_destroy( &serve->l_acl ); + pthread_mutex_destroy( &serve->l_io ); + + if ( serve->acl ) { acl_destroy( serve->acl ); } + + free( serve ); +} + + void server_dirty(struct server *serve, off64_t from, int len) { NULLCHECK( serve ); @@ -70,6 +139,23 @@ void server_unlock_acl( struct server *serve ) } +/** Return the actual port the server bound to. This is used because we + * are allowed to pass "0" on the command-line. + */ +int server_port( struct server * server ) +{ + NULLCHECK( server ); + union mysockaddr addr; + socklen_t len = sizeof( addr.v4 ); + + if ( getsockname( server->server_fd, &addr.v4, &len ) < 0 ) { + SERVER_ERROR( "Failed to get the port number." ); + } + + return be16toh( addr.v4.sin_port ); +} + + /** Prepares a listening socket for the NBD server, binding etc. */ void serve_open_server_socket(struct server* params) { @@ -105,6 +191,8 @@ void serve_open_server_socket(struct server* params) ); } + + int tryjoin_client_thread( struct client_tbl_entry *entry, int (*joinfunc)(pthread_t, void **) ) { @@ -290,7 +378,7 @@ void accept_nbd_client( memcpy(¶ms->nbd_client[slot].address, client_address, sizeof(union mysockaddr)); - if (pthread_create(¶ms->nbd_client[slot].thread, NULL, client_serve, client_params) < 0) { + if (pthread_create(¶ms->nbd_client[slot].thread, NULL, client_serve, client_params) != 0) { debug( "Thread creation problem." ); write(client_fd, "Thread creation problem", 23); client_destroy( client_params ); @@ -302,6 +390,30 @@ void accept_nbd_client( } +void server_audit_clients( struct server * serve) +{ + NULLCHECK( serve ); + + int i; + struct client_tbl_entry * entry; + + /* There's an apparent race here. If the acl updates while + * we're traversing the nbd_clients array, the earlier entries + * won't have been audited against the later acl. This isn't a + * problem though, because in order to update the acl + * server_replace_acl must have been called, so the + * server_accept loop will see a second acl_updated signal as + * soon as it hits select, and a second audit will be run. + */ + for( i = 0; i < MAX_NBD_CLIENTS; i++ ) { + entry = &serve->nbd_client[i]; + if ( 0 == entry->thread ) { continue; } + if ( server_acl_accepts( serve, &entry->address ) ) { continue; } + client_signal_stop( entry->client ); + } +} + + int server_is_closed(struct server* serve) { NULLCHECK( serve ); @@ -329,6 +441,9 @@ void server_close_clients( struct server *params ) } +/** Replace the current acl with a new one. The old one will be thrown + * away. + */ void server_replace_acl( struct server *serve, struct acl * new_acl ) { NULLCHECK(serve); @@ -351,43 +466,54 @@ void server_replace_acl( struct server *serve, struct acl * new_acl ) /** Accept either an NBD or control socket connection, dispatch appropriately */ -void serve_accept_loop(struct server* params) +int server_accept( struct server * params ) { NULLCHECK( params ); - while (1) { - int activity_fd, client_fd; - union mysockaddr client_address; - fd_set fds; - socklen_t socklen=sizeof(client_address); - - FD_ZERO(&fds); - FD_SET(params->server_fd, &fds); - self_pipe_fd_set( params->close_signal, &fds ); - if (params->control_socket_name) - FD_SET(params->control_fd, &fds); - - SERVER_ERROR_ON_FAILURE(select(FD_SETSIZE, &fds, - NULL, NULL, NULL), "select() failed"); - - if ( self_pipe_fd_isset( params->close_signal, &fds ) ){ - server_close_clients( params ); - return; - } - - activity_fd = FD_ISSET(params->server_fd, &fds) ? params->server_fd: - params->control_fd; - client_fd = accept(activity_fd, &client_address.generic, &socklen); - - if (activity_fd == params->server_fd) { - debug("Accepted nbd client socket"); - accept_nbd_client(params, client_fd, &client_address); - } - if (activity_fd == params->control_fd) { - debug("Accepted control client socket"); - accept_control_connection(params, client_fd, &client_address); - } - + + int activity_fd, client_fd; + union mysockaddr client_address; + fd_set fds; + socklen_t socklen=sizeof(client_address); + + FD_ZERO(&fds); + FD_SET(params->server_fd, &fds); + self_pipe_fd_set( params->close_signal, &fds ); + self_pipe_fd_set( params->acl_updated_signal, &fds ); + if (params->control_socket_name) + FD_SET(params->control_fd, &fds); + + SERVER_ERROR_ON_FAILURE(select(FD_SETSIZE, &fds, + NULL, NULL, NULL), "select() failed"); + + if ( self_pipe_fd_isset( params->close_signal, &fds ) ){ + server_close_clients( params ); + return 0; } + + if ( self_pipe_fd_isset( params->acl_updated_signal, &fds ) ) { + server_audit_clients( params ); + } + + activity_fd = FD_ISSET(params->server_fd, &fds) ? params->server_fd: + params->control_fd; + client_fd = accept(activity_fd, &client_address.generic, &socklen); + + if (activity_fd == params->server_fd) { + debug("Accepted nbd client socket"); + accept_nbd_client(params, client_fd, &client_address); + } + if (activity_fd == params->control_fd) { + debug("Accepted control client socket"); + accept_control_connection(params, client_fd, &client_address); + } + + return 1; +} + + +void serve_accept_loop(struct server* params) +{ + while( server_accept( params ) ); } /** Initialisation function that sets up the initial allocation map, i.e. so @@ -404,7 +530,7 @@ void serve_init_allocation_map(struct server* params) size = lseek64(fd, 0, SEEK_END); params->size = size; SERVER_ERROR_ON_FAILURE(size, "Couldn't find size of %s", - params->filename); + params->filename); params->allocation_map = build_allocation_map(fd, size, block_allocation_resolution); close(fd); @@ -428,18 +554,12 @@ void serve_cleanup(struct server* params) close(params->server_fd); close(params->control_fd); - if (params->acl) - free(params->acl); - //free(params->filename); - if (params->control_socket_name) + if (params->control_socket_name){ //free(params->control_socket_name); - pthread_mutex_destroy(¶ms->l_io); + } if (params->proxy_fd); close(params->proxy_fd); - self_pipe_destroy( params->close_signal ); - self_pipe_destroy( params->acl_updated_signal ); - free(params->allocation_map); if (params->mirror) @@ -460,14 +580,6 @@ void do_serve(struct server* params) { NULLCHECK( params ); - pthread_mutex_init(¶ms->l_io, NULL); - pthread_mutex_init(¶ms->l_acl, NULL); - - params->close_signal = self_pipe_create(); - NULLCHECK( params->close_signal ); - params->acl_updated_signal = self_pipe_create(); - NULLCHECK( params->acl_updated_signal ); - serve_open_server_socket(params); serve_open_control_socket(params); serve_init_allocation_map(params); diff --git a/src/serve.h b/src/serve.h index 35118c7..210a8cb 100644 --- a/src/serve.h +++ b/src/serve.h @@ -84,11 +84,15 @@ struct server { struct client_tbl_entry nbd_client[MAX_NBD_CLIENTS]; }; +struct server * server_create( char* s_ip_address, char* s_port, char* s_file, + char *s_ctrl_sock, int default_deny, int acl_entries, char** s_acl_entries ); +void server_destroy( struct server * ); int server_is_closed(struct server* serve); void server_dirty(struct server *serve, off64_t from, int len); void server_lock_io( struct server * serve); void server_unlock_io( struct server* serve ); void serve_signal_close( struct server *serve ); +void server_replace_acl( struct server *serve, struct acl * acl); struct mode_readwrite_params { diff --git a/tests/check_serve.c b/tests/check_serve.c index 823bcfb..8877d20 100644 --- a/tests/check_serve.c +++ b/tests/check_serve.c @@ -1,41 +1,197 @@ #include "serve.h" #include "self_pipe.h" +#include "client.h" +#include #include #include +#include +#include +#include +#include +#include + + +char * dummy_file; + +char *make_tmpfile() +{ + FILE *fp; + char *fn_buf; + char leader[] = "/tmp/check_serve"; + + fn_buf = (char *)malloc( 1024 ); + strncpy( fn_buf, leader, sizeof( leader ) - 1); + snprintf( &fn_buf[sizeof( leader ) - 1], 10, "%d", getpid() ); + fp = fopen( fn_buf, "w" ); + fwrite( fn_buf, 1024, 1, fp ); + fclose( fp ); + + return fn_buf; +} + + +void setup( void ) +{ + dummy_file = make_tmpfile(); +} + +void teardown( void ) +{ + if( dummy_file ){ unlink( dummy_file ); } + free( dummy_file ); + dummy_file = NULL; +} + +/* Need these because libcheck is braindead and doesn't + * run teardown after a failing test + */ +#define myfail( msg ) do { teardown(); fail(msg); } while (0) +#define myfail_if( tst, msg ) do { if( tst ) { myfail( msg ); } } while (0) +#define myfail_unless( tst, msg ) myfail_if( !(tst), msg ) START_TEST( test_replaces_acl ) { - struct server s = {0}; - s.acl_updated_signal = self_pipe_create(); - pthread_mutex_init( &s.l_acl, NULL ); + struct server * s = server_create( "127.0.0.1", "0", dummy_file, NULL, 0, 0, NULL ); + struct acl * new_acl = acl_create( 0, NULL, 0 ); - struct acl * acl = acl_create( 0, NULL, 0 ); + server_replace_acl( s, new_acl ); - server_replace_acl( &s, acl ); - - fail_unless( s.acl == acl, "ACL wasn't replaced." ); - self_pipe_destroy( s.acl_updated_signal ); + myfail_unless( s->acl == new_acl, "ACL wasn't replaced." ); + server_destroy( s ); } END_TEST START_TEST( test_signals_acl_updated ) { - struct server s = {0}; + struct server * s = server_create( "127.0.0.1", "0", dummy_file, NULL, 0, 0, NULL ); struct acl * new_acl = acl_create( 0, NULL, 0 ); - s.acl_updated_signal = self_pipe_create(); - pthread_mutex_init( &s.l_acl, NULL ); - s.acl = acl_create( 0, NULL, 0); + server_replace_acl( s, new_acl ); - - server_replace_acl( &s, new_acl ); - - fail_unless( 1 == self_pipe_signal_clear( s.acl_updated_signal ), + myfail_unless( 1 == self_pipe_signal_clear( s->acl_updated_signal ), "No signal sent." ); - self_pipe_destroy( s.acl_updated_signal ); + server_destroy( s ); +} +END_TEST + + +int connect_client( char *addr, int actual_port ) +{ + int client_fd; + + struct addrinfo hint = {0}; + struct addrinfo *ailist, *aip; + hint.ai_socktype = SOCK_STREAM; + + myfail_if( getaddrinfo( "127.0.0.7", NULL, &hint, &ailist ) != 0, "getaddrinfo failed." ); + + int connected = 0; + for( aip = ailist; aip; aip = aip->ai_next ) { + ((struct sockaddr_in *)aip->ai_addr)->sin_port = htons( actual_port ); + client_fd = socket( aip->ai_family, aip->ai_socktype, aip->ai_protocol ); + if( client_fd == -1) { continue; } + if( connect( client_fd, aip->ai_addr, aip->ai_addrlen) == 0 ) { + connected = 1; + break; + } + close( client_fd ); + } + + myfail_unless( connected, "Didn't connect." ); + return client_fd; +} + + +START_TEST( test_acl_update_closes_bad_client ) +{ + /* This is the wrong way round. Rather than pulling the thread + * and socket out of the server structure, we should be testing + * a client socket. + */ + struct server * s = server_create( "127.0.0.7", "0", dummy_file, NULL, 0, 0, NULL ); + struct acl * new_acl = acl_create( 0, NULL, 1 ); + struct client * c; + struct client_tbl_entry * entry; + + int actual_port; + int client_fd; + int server_fd; + + + serve_open_server_socket( s ); + actual_port = server_port( s ); + + client_fd = connect_client( "127.0.0.7", actual_port ); + server_accept( s ); + entry = &s->nbd_client[0]; + c = entry->client; + /* At this point there should be an entry in the nbd_clients + * table and a background thread to run the client loop + */ + myfail_if( entry->thread == 0, "No client thread was started." ); + server_fd = c->socket; + myfail_if( fd_is_closed(server_fd), + "Sanity check failed - client socket wasn't open." ); + + server_replace_acl( s, new_acl ); + + server_accept( s ); + + pthread_join( entry->thread ); + + myfail_unless( fd_is_closed(server_fd), + "Client socket wasn't closed." ); + close( client_fd ); + server_close_clients( s ); + server_destroy( s ); +} +END_TEST + + +START_TEST( test_acl_update_leaves_good_client ) +{ + struct server * s = server_create( "127.0.0.7", "0", dummy_file, NULL, 0, 0, NULL ); + /* There's an assumption here that the localhost *is* 127.0.0.1. + * If it's not, this test will fail and we'll have to explicitly + * pick a source address. + */ + char *lines[] = {"127.0.0.1"}; + struct acl * new_acl = acl_create( 1, lines, 0); + struct client * c; + struct client_tbl_entry * entry; + + int actual_port; + int client_fd; + int server_fd; + + + serve_open_server_socket( s ); + actual_port = server_port( s ); + + client_fd = connect_client( "127.0.0.7", actual_port ); + server_accept( s ); + entry = &s->nbd_client[0]; + c = entry->client; + /* At this point there should be an entry in the nbd_clients + * table and a background thread to run the client loop + */ + myfail_if( entry->thread == 0, "No client thread was started." ); + server_fd = c->socket; + myfail_if( fd_is_closed(server_fd), + "Sanity check failed - client socket wasn't open." ); + + server_replace_acl( s, new_acl ); + server_accept( s ); + + myfail_if( self_pipe_signal_clear( c->stop_signal ), + "Client was told to stop." ); + + close( client_fd ); + server_close_clients( s ); + server_destroy( s ); } END_TEST @@ -45,8 +201,12 @@ Suite* serve_suite() Suite *s = suite_create("serve"); TCase *tc_acl_update = tcase_create("acl_update"); + tcase_add_checked_fixture( tc_acl_update, setup, teardown ); tcase_add_test(tc_acl_update, test_replaces_acl); tcase_add_test(tc_acl_update, test_signals_acl_updated); + tcase_add_test(tc_acl_update, test_acl_update_closes_bad_client); + + tcase_add_test(tc_acl_update, test_acl_update_leaves_good_client); suite_add_tcase(s, tc_acl_update);