flexnbd: Add a proxy mode
This lets us proxy connections between NBD clients and servers, resiliently.
This commit is contained in:
@@ -129,6 +129,26 @@ struct flexnbd * flexnbd_create_listening(
|
||||
}
|
||||
|
||||
|
||||
struct flexnbd * flexnbd_create_proxying(
|
||||
char* s_downstream_address,
|
||||
char* s_downstream_port,
|
||||
char* s_upstream_address,
|
||||
char* s_upstream_port,
|
||||
char* s_upstream_bind
|
||||
)
|
||||
{
|
||||
struct flexnbd * flexnbd = xmalloc( sizeof( struct flexnbd ) );
|
||||
flexnbd->proxy = proxy_create(
|
||||
flexnbd,
|
||||
s_downstream_address,
|
||||
s_downstream_port,
|
||||
s_upstream_address,
|
||||
s_upstream_port,
|
||||
s_upstream_bind);
|
||||
flexnbd_create_shared( flexnbd, NULL );
|
||||
return flexnbd;
|
||||
}
|
||||
|
||||
void flexnbd_spawn_control(struct flexnbd * flexnbd )
|
||||
{
|
||||
NULLCHECK( flexnbd );
|
||||
@@ -181,7 +201,6 @@ struct server * flexnbd_server( struct flexnbd * flexnbd )
|
||||
return flexnbd->serve;
|
||||
}
|
||||
|
||||
|
||||
void flexnbd_replace_acl( struct flexnbd * flexnbd, struct acl * acl )
|
||||
{
|
||||
NULLCHECK( flexnbd );
|
||||
@@ -255,3 +274,14 @@ int flexnbd_serve( struct flexnbd * flexnbd )
|
||||
return success;
|
||||
}
|
||||
|
||||
int flexnbd_proxy( struct flexnbd * flexnbd )
|
||||
{
|
||||
NULLCHECK( flexnbd );
|
||||
int success;
|
||||
|
||||
success = do_proxy( flexnbd->proxy );
|
||||
debug("do_proxy success is %d", success );
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
|
@@ -4,6 +4,7 @@
|
||||
#include "acl.h"
|
||||
#include "mirror.h"
|
||||
#include "serve.h"
|
||||
#include "proxy.h"
|
||||
#include "self_pipe.h"
|
||||
#include "mbox.h"
|
||||
#include "control.h"
|
||||
@@ -11,11 +12,14 @@
|
||||
|
||||
/* Carries the "globals". */
|
||||
struct flexnbd {
|
||||
/* We always have a serve pointer, but it should never be
|
||||
* dereferenced outside a flexnbd_switch_lock/unlock pair.
|
||||
/* Our serve pointer should never be dereferenced outside a
|
||||
* flexnbd_switch_lock/unlock pair.
|
||||
*/
|
||||
struct server * serve;
|
||||
|
||||
/* In proxy mode, this is filled instead of serve, above */
|
||||
struct proxier * proxy;
|
||||
|
||||
/* We only have a control object if a control socket name was
|
||||
* passed on the command line.
|
||||
*/
|
||||
@@ -46,6 +50,14 @@ struct flexnbd * flexnbd_create_listening(
|
||||
int acl_entries,
|
||||
char** s_acl_entries );
|
||||
|
||||
struct flexnbd * flexnbd_create_proxying(
|
||||
char* s_downstream_address,
|
||||
char* s_downstream_port,
|
||||
char* s_upstream_address,
|
||||
char* s_upstream_port,
|
||||
char* s_upstream_bind
|
||||
);
|
||||
|
||||
void flexnbd_destroy( struct flexnbd * );
|
||||
enum mirror_state;
|
||||
enum mirror_state flexnbd_get_mirror_state( struct flexnbd * );
|
||||
@@ -55,7 +67,9 @@ int flexnbd_signal_fd( struct flexnbd * flexnbd );
|
||||
|
||||
|
||||
int flexnbd_serve( struct flexnbd * flexnbd );
|
||||
int flexnbd_proxy( struct flexnbd * flexnbd );
|
||||
struct server * flexnbd_server( struct flexnbd * flexnbd );
|
||||
void flexnbd_replace_acl( struct flexnbd * flexnbd, struct acl * acl );
|
||||
struct status * flexnbd_status_create( struct flexnbd * flexnbd );
|
||||
#endif
|
||||
|
||||
|
120
src/mode.c
120
src/mode.c
@@ -56,6 +56,29 @@ static char listen_help_text[] =
|
||||
VERBOSE_LINE
|
||||
QUIET_LINE;
|
||||
|
||||
static struct option proxy_options[] = {
|
||||
GETOPT_HELP,
|
||||
GETOPT_ADDR,
|
||||
GETOPT_PORT,
|
||||
GETOPT_CONNECT_ADDR,
|
||||
GETOPT_CONNECT_PORT,
|
||||
GETOPT_BIND,
|
||||
GETOPT_QUIET,
|
||||
GETOPT_VERBOSE,
|
||||
{0}
|
||||
};
|
||||
static char proxy_short_options[] = "hl:p:C:P:b:" SOPT_QUIET SOPT_VERBOSE;
|
||||
static char proxy_help_text[] =
|
||||
"Usage: flexnbd " CMD_PROXY " <options>\n\n"
|
||||
"Resiliently proxy an NBD connection between client and server\n\n"
|
||||
HELP_LINE
|
||||
"\t--" OPT_ADDR ",-l <ADDR>\tThe address we will bind to as a proxy.\n"
|
||||
"\t--" OPT_PORT ",-p <PORT>\tThe port we will bind to as a proxy.\n"
|
||||
"\t--" OPT_CONNECT_ADDR ",-C <ADDR>\tAddress of the proxied server.\n"
|
||||
"\t--" OPT_CONNECT_PORT ",-P <PORT>\tPort of the proxied server.\n"
|
||||
"\t--" OPT_BIND ",-b <ADDR>\tThe address we connect from, as a proxy.\n"
|
||||
QUIET_LINE
|
||||
VERBOSE_LINE;
|
||||
|
||||
static struct option read_options[] = {
|
||||
GETOPT_HELP,
|
||||
@@ -173,10 +196,13 @@ char help_help_text_arr[] =
|
||||
"Usage: flexnbd <cmd> [cmd options]\n\n"
|
||||
"Commands:\n"
|
||||
"\tflexnbd serve\n"
|
||||
"\tflexnbd listen\n"
|
||||
"\tflexnbd proxy\n"
|
||||
"\tflexnbd read\n"
|
||||
"\tflexnbd write\n"
|
||||
"\tflexnbd acl\n"
|
||||
"\tflexnbd mirror\n"
|
||||
"\tflexnbd break\n"
|
||||
"\tflexnbd status\n"
|
||||
"\tflexnbd help\n\n"
|
||||
"See flexnbd help <cmd> for further info\n";
|
||||
@@ -390,6 +416,46 @@ void read_break_param( int c, char **sock )
|
||||
}
|
||||
|
||||
|
||||
void read_proxy_param(
|
||||
int c,
|
||||
char **downstream_addr,
|
||||
char **downstream_port,
|
||||
char **upstream_addr,
|
||||
char **upstream_port,
|
||||
char **bind_addr )
|
||||
{
|
||||
switch( c ) {
|
||||
case 'h' :
|
||||
fprintf( stdout, "%s\n", proxy_help_text );
|
||||
exit( 0 );
|
||||
break;
|
||||
case 'l':
|
||||
*downstream_addr = optarg;
|
||||
break;
|
||||
case 'p':
|
||||
*downstream_port = optarg;
|
||||
break;
|
||||
case 'C':
|
||||
*upstream_addr = optarg;
|
||||
break;
|
||||
case 'P':
|
||||
*upstream_port = optarg;
|
||||
break;
|
||||
case 'b':
|
||||
*bind_addr = optarg;
|
||||
break;
|
||||
case 'q':
|
||||
log_level = QUIET_LOG_LEVEL;
|
||||
break;
|
||||
case 'v':
|
||||
log_level = VERBOSE_LOG_LEVEL;
|
||||
break;
|
||||
default:
|
||||
exit_err( proxy_help_text );
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void read_status_param( int c, char **sock )
|
||||
{
|
||||
read_sock_param( c, sock, status_help_text );
|
||||
@@ -733,6 +799,55 @@ int mode_status( int argc, char *argv[] )
|
||||
return 0;
|
||||
}
|
||||
|
||||
int mode_proxy( int argc, char *argv[] )
|
||||
{
|
||||
int c;
|
||||
struct flexnbd * flexnbd;
|
||||
char *downstream_addr = NULL;
|
||||
char *downstream_port = NULL;
|
||||
char *upstream_addr = NULL;
|
||||
char *upstream_port = NULL;
|
||||
char *bind_addr = NULL;
|
||||
int success;
|
||||
|
||||
while (1) {
|
||||
c = getopt_long( argc, argv, proxy_short_options, proxy_options, NULL );
|
||||
if ( -1 == c ) { break; }
|
||||
read_proxy_param( c,
|
||||
&downstream_addr,
|
||||
&downstream_port,
|
||||
&upstream_addr,
|
||||
&upstream_port,
|
||||
&bind_addr
|
||||
);
|
||||
}
|
||||
|
||||
if ( NULL == downstream_addr || NULL == downstream_port ){
|
||||
fprintf( stderr, "both --addr and --port are required.\n" );
|
||||
exit_err( proxy_help_text );
|
||||
} else if ( NULL == upstream_addr || NULL == upstream_port ){
|
||||
fprintf( stderr, "both --conn-addr and --conn-port are required.\n" );
|
||||
exit_err( proxy_help_text );
|
||||
}
|
||||
|
||||
flexnbd = flexnbd_create_proxying(
|
||||
downstream_addr,
|
||||
downstream_port,
|
||||
upstream_addr,
|
||||
upstream_port,
|
||||
bind_addr
|
||||
);
|
||||
|
||||
info(
|
||||
"Proxying between %s %s (downstream) and %s %s (upstream)",
|
||||
downstream_addr, downstream_port, upstream_addr, upstream_port
|
||||
);
|
||||
|
||||
success = flexnbd_proxy( flexnbd );
|
||||
flexnbd_destroy( flexnbd );
|
||||
|
||||
return success ? 0 : 1;
|
||||
}
|
||||
|
||||
int mode_help( int argc, char *argv[] )
|
||||
{
|
||||
@@ -757,6 +872,8 @@ int mode_help( int argc, char *argv[] )
|
||||
help_text = mirror_help_text;
|
||||
} else if ( IS_CMD( CMD_STATUS, cmd ) ) {
|
||||
help_text = status_help_text;
|
||||
} else if ( IS_CMD( CMD_PROXY, cmd ) ) {
|
||||
help_text = proxy_help_text;
|
||||
} else { exit_err( help_help_text ); }
|
||||
}
|
||||
|
||||
@@ -790,6 +907,8 @@ void mode(char* mode, int argc, char **argv)
|
||||
}
|
||||
else if ( IS_CMD( CMD_STATUS, mode ) ) {
|
||||
mode_status( argc, argv );
|
||||
} else if ( IS_CMD( CMD_PROXY, mode ) ) {
|
||||
mode_proxy( argc, argv );
|
||||
}
|
||||
else if ( IS_CMD( CMD_HELP, mode ) ) {
|
||||
mode_help( argc-1, argv+1 );
|
||||
@@ -801,4 +920,3 @@ void mode(char* mode, int argc, char **argv)
|
||||
exit(0);
|
||||
}
|
||||
|
||||
|
||||
|
@@ -20,9 +20,12 @@ void mode(char* mode, int argc, char **argv);
|
||||
#define OPT_SIZE "size"
|
||||
#define OPT_DENY "default-deny"
|
||||
#define OPT_UNLINK "unlink"
|
||||
#define OPT_CONNECT_ADDR "conn-addr"
|
||||
#define OPT_CONNECT_PORT "conn-port"
|
||||
|
||||
#define CMD_SERVE "serve"
|
||||
#define CMD_LISTEN "listen"
|
||||
#define CMD_PROXY "proxy"
|
||||
#define CMD_READ "read"
|
||||
#define CMD_WRITE "write"
|
||||
#define CMD_ACL "acl"
|
||||
@@ -40,7 +43,6 @@ void mode(char* mode, int argc, char **argv);
|
||||
|
||||
#define GETOPT_HELP GETOPT_FLAG( OPT_HELP, 'h' )
|
||||
#define GETOPT_DENY GETOPT_FLAG( OPT_DENY, 'd' )
|
||||
|
||||
#define GETOPT_ADDR GETOPT_ARG( OPT_ADDR, 'l' )
|
||||
#define GETOPT_PORT GETOPT_ARG( OPT_PORT, 'p' )
|
||||
#define GETOPT_FILE GETOPT_ARG( OPT_FILE, 'f' )
|
||||
@@ -49,6 +51,8 @@ void mode(char* mode, int argc, char **argv);
|
||||
#define GETOPT_SIZE GETOPT_ARG( OPT_SIZE, 'S' )
|
||||
#define GETOPT_BIND GETOPT_ARG( OPT_BIND, 'b' )
|
||||
#define GETOPT_UNLINK GETOPT_ARG( OPT_UNLINK, 'u' )
|
||||
#define GETOPT_CONNECT_ADDR GETOPT_ARG( OPT_CONNECT_ADDR, 'C' )
|
||||
#define GETOPT_CONNECT_PORT GETOPT_ARG( OPT_CONNECT_PORT, 'P' )
|
||||
|
||||
#define OPT_VERBOSE "verbose"
|
||||
#define SOPT_VERBOSE "v"
|
||||
|
497
src/proxy.c
Normal file
497
src/proxy.c
Normal file
@@ -0,0 +1,497 @@
|
||||
#include "proxy.h"
|
||||
#include "readwrite.h"
|
||||
|
||||
#include "ioutil.h"
|
||||
#include "sockutil.h"
|
||||
#include "util.h"
|
||||
|
||||
#include <errno.h>
|
||||
|
||||
#include <sys/socket.h>
|
||||
#include <netinet/tcp.h>
|
||||
|
||||
struct proxier* proxy_create(
|
||||
struct flexnbd* flexnbd,
|
||||
char* s_downstream_address,
|
||||
char* s_downstream_port,
|
||||
char* s_upstream_address,
|
||||
char* s_upstream_port,
|
||||
char* s_upstream_bind )
|
||||
{
|
||||
NULLCHECK( flexnbd );
|
||||
|
||||
struct proxier* out;
|
||||
out = xmalloc( sizeof( struct proxier ) );
|
||||
out->flexnbd = flexnbd;
|
||||
|
||||
FATAL_IF_NULL(s_downstream_address, "Listen address not specified");
|
||||
NULLCHECK( s_downstream_address );
|
||||
|
||||
FATAL_UNLESS(
|
||||
parse_ip_to_sockaddr( &out->listen_on.generic, s_downstream_address ),
|
||||
"Couldn't parse downstream address '%s' (use 0 if "
|
||||
"you want to bind all IPs)",
|
||||
s_downstream_address
|
||||
);
|
||||
|
||||
FATAL_IF_NULL( s_downstream_port, "Downstream port not specified" );
|
||||
NULLCHECK( s_downstream_port );
|
||||
parse_port( s_downstream_port, &out->listen_on.v4 );
|
||||
|
||||
FATAL_IF_NULL(s_upstream_address, "Upstream address not specified");
|
||||
NULLCHECK( s_upstream_address );
|
||||
|
||||
FATAL_UNLESS(
|
||||
parse_ip_to_sockaddr( &out->connect_to.generic, s_upstream_address ),
|
||||
"Couldn't parse upstream address '%s'",
|
||||
s_upstream_address
|
||||
);
|
||||
|
||||
FATAL_IF_NULL( s_upstream_port, "Upstream port not specified" );
|
||||
NULLCHECK( s_upstream_port );
|
||||
parse_port( s_upstream_port, &out->connect_to.v4 );
|
||||
|
||||
if ( s_upstream_bind ) {
|
||||
FATAL_IF_ZERO(
|
||||
parse_ip_to_sockaddr( &out->connect_from.generic, s_upstream_bind ),
|
||||
"Couldn't parse bind address '%s'",
|
||||
s_upstream_bind
|
||||
);
|
||||
}
|
||||
|
||||
out->listen_fd = -1;
|
||||
out->downstream_fd = -1;
|
||||
out->upstream_fd = -1;
|
||||
|
||||
out->req_buf = xmalloc( NBD_MAX_SIZE );
|
||||
out->rsp_buf = xmalloc( NBD_MAX_SIZE );
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
void proxy_destroy( struct proxier* proxy )
|
||||
{
|
||||
free( proxy->req_buf );
|
||||
free( proxy->rsp_buf );
|
||||
free( proxy );
|
||||
}
|
||||
|
||||
|
||||
/* Try to establish a connection to our upstream server. Return 1 on success,
|
||||
* 0 on failure
|
||||
*/
|
||||
int proxy_connect_to_upstream( struct proxier* proxy )
|
||||
{
|
||||
int fd = socket_connect( &proxy->connect_to.generic, &proxy->connect_from.generic );
|
||||
off64_t size = 0;
|
||||
|
||||
if ( -1 == fd ) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if( !socket_nbd_read_hello( fd, &size ) ) {
|
||||
FATAL_IF_NEGATIVE(
|
||||
close( fd ), SHOW_ERRNO( "FIXME: shouldn't be fatal" )
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if ( proxy->upstream_size == 0 ) {
|
||||
info( "Size of upstream image is %"PRIu64" bytes", size );
|
||||
} else if ( proxy->upstream_size != size ) {
|
||||
warn( "Size changed from %"PRIu64" to %"PRIu64" bytes", proxy->upstream_size, size );
|
||||
}
|
||||
|
||||
proxy->upstream_size = size;
|
||||
proxy->upstream_fd = fd;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
void proxy_disconnect_from_upstream( struct proxier* proxy )
|
||||
{
|
||||
if ( -1 != proxy->upstream_fd ) {
|
||||
debug(" Closing upstream connection" );
|
||||
|
||||
/* TODO: An NBD disconnect would be pleasant here */
|
||||
|
||||
FATAL_IF_NEGATIVE(
|
||||
close( proxy->upstream_fd ),
|
||||
SHOW_ERRNO( "FIXME: shouldn't be fatal" )
|
||||
);
|
||||
proxy->upstream_fd = -1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** Prepares a listening socket for the NBD server, binding etc. */
|
||||
void proxy_open_listen_socket(struct proxier* params)
|
||||
{
|
||||
NULLCHECK( params );
|
||||
|
||||
params->listen_fd = socket(params->listen_on.family, SOCK_STREAM, 0);
|
||||
FATAL_IF_NEGATIVE(
|
||||
params->listen_fd, SHOW_ERRNO( "Couldn't create listen socket" )
|
||||
);
|
||||
|
||||
/* Allow us to restart quickly */
|
||||
FATAL_IF_NEGATIVE(
|
||||
sock_set_reuseaddr(params->listen_fd, 1),
|
||||
SHOW_ERRNO( "Couldn't set SO_REUSEADDR" )
|
||||
);
|
||||
|
||||
FATAL_IF_NEGATIVE(
|
||||
sock_set_tcp_nodelay(params->listen_fd, 1),
|
||||
SHOW_ERRNO( "Couldn't set TCP_NODELAY" )
|
||||
);
|
||||
|
||||
FATAL_UNLESS_ZERO(
|
||||
sock_try_bind( params->listen_fd, ¶ms->listen_on.generic ),
|
||||
SHOW_ERRNO( "Failed to bind to listening socket" )
|
||||
);
|
||||
|
||||
/* We're only serving one client at a time, hence backlog of 1 */
|
||||
FATAL_IF_NEGATIVE(
|
||||
listen(params->listen_fd, 1),
|
||||
SHOW_ERRNO( "Failed to listen on listening socket" )
|
||||
);
|
||||
|
||||
info( "Now listening for incoming connections" );
|
||||
}
|
||||
|
||||
|
||||
/* Return 0 if we should keep running, 1 if an exit has been signaled. Pass it
|
||||
* an fd_set to check, or set check_fds to NULL to have it perform its own.
|
||||
* If we do the latter, then wait specifies how many seconds we'll wait for an
|
||||
* exit signal to show up.
|
||||
*/
|
||||
int proxy_should_exit( struct proxier* params, fd_set *check_fds, int wait )
|
||||
{
|
||||
struct timeval tv = { wait, 0 };
|
||||
fd_set internal_fds;
|
||||
fd_set* fds = check_fds;
|
||||
|
||||
int signal_fd = flexnbd_signal_fd( params->flexnbd );
|
||||
|
||||
if ( NULL == check_fds ) {
|
||||
fds = &internal_fds;
|
||||
|
||||
FD_ZERO( fds );
|
||||
FD_SET( signal_fd, fds );
|
||||
|
||||
FATAL_IF_NEGATIVE(
|
||||
sock_try_select(FD_SETSIZE, fds, NULL, NULL, &tv),
|
||||
SHOW_ERRNO( "select() failed." )
|
||||
);
|
||||
}
|
||||
|
||||
if ( FD_ISSET( signal_fd, fds ) ) {
|
||||
info( "Stop signal received" );
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Try to get a request from downstream. If reading from downstream fails, then
|
||||
* the session will be over. Returns 1 on success, 0 on failure.
|
||||
*/
|
||||
int proxy_get_request_from_downstream( struct proxier* proxy )
|
||||
{
|
||||
unsigned char* req_hdr_raw = proxy->req_buf;
|
||||
unsigned char* req_data = proxy->req_buf + NBD_REQUEST_SIZE;
|
||||
size_t req_buf_size;
|
||||
|
||||
struct nbd_request_raw* request_raw = (struct nbd_request_raw*) req_hdr_raw;
|
||||
struct nbd_request* request = &(proxy->req_hdr);
|
||||
|
||||
if ( readloop( proxy->downstream_fd, req_hdr_raw, NBD_REQUEST_SIZE ) == -1 ) {
|
||||
info( SHOW_ERRNO( "Failed to get request header" ) );
|
||||
return 0;
|
||||
}
|
||||
|
||||
nbd_r2h_request( request_raw, request );
|
||||
req_buf_size = NBD_REQUEST_SIZE;
|
||||
|
||||
if ( request->type == REQUEST_DISCONNECT ) {
|
||||
info( "Received disconnect request from client" );
|
||||
return 0;
|
||||
}
|
||||
|
||||
if ( request->type == REQUEST_READ ) {
|
||||
if (request->len > ( NBD_MAX_SIZE - NBD_REPLY_SIZE ) ) {
|
||||
warn( "NBD read request size %"PRIu32" too large", request->len );
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if ( request->type == REQUEST_WRITE ) {
|
||||
if (request->len > ( NBD_MAX_SIZE - NBD_REQUEST_SIZE ) ) {
|
||||
warn( "NBD write request size %"PRIu32" too large", request->len );
|
||||
return 0;
|
||||
}
|
||||
|
||||
if ( readloop( proxy->downstream_fd, req_data, request->len ) == -1 ) {
|
||||
warn( "Failed to get NBD write request data: %"PRIu32"b", request->len );
|
||||
return 0;
|
||||
}
|
||||
|
||||
req_buf_size += request->len;
|
||||
}
|
||||
|
||||
debug(
|
||||
"Received NBD request from downstream. type=%"PRIu32" from=%"PRIu64" len=%"PRIu32,
|
||||
request->type, request->from, request->len
|
||||
);
|
||||
|
||||
proxy->req_buf_size = req_buf_size;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Tries to send the request upstream and receive a response. If upstream breaks
|
||||
* then we reconnect to it, and keep it up until we have a complete response
|
||||
* back. Returns 1 on success, 0 on failure, -1 if exit is signalled.
|
||||
*/
|
||||
int proxy_run_request_upstream( struct proxier* proxy )
|
||||
{
|
||||
unsigned char* rsp_hdr_raw = proxy->rsp_buf;
|
||||
unsigned char* rsp_data = proxy->rsp_buf + NBD_REPLY_SIZE;
|
||||
|
||||
struct nbd_reply_raw* reply_raw = (struct nbd_reply_raw*) rsp_hdr_raw;
|
||||
|
||||
struct nbd_request* request = &(proxy->req_hdr);
|
||||
struct nbd_reply* reply = &(proxy->rsp_hdr);
|
||||
|
||||
size_t rsp_buf_size;
|
||||
|
||||
if ( proxy->upstream_fd == -1 ) {
|
||||
debug( "Connecting to upstream" );
|
||||
if ( !proxy_connect_to_upstream( proxy ) ) {
|
||||
debug( "Failed to connect to upstream" );
|
||||
|
||||
if ( proxy_should_exit( proxy, NULL, 5 ) ) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
debug( "Connected to upstream" );
|
||||
}
|
||||
|
||||
if ( writeloop( proxy->upstream_fd, proxy->req_buf, proxy->req_buf_size ) == -1 ) {
|
||||
warn( "Failed to send request to upstream" );
|
||||
proxy_disconnect_from_upstream( proxy );
|
||||
return 0;
|
||||
}
|
||||
|
||||
if ( readloop( proxy->upstream_fd, rsp_hdr_raw, NBD_REPLY_SIZE ) == -1 ) {
|
||||
debug( "Failed to get reply header from upstream" );
|
||||
proxy_disconnect_from_upstream( proxy );
|
||||
return 0;
|
||||
}
|
||||
|
||||
nbd_r2h_reply( reply_raw, reply );
|
||||
rsp_buf_size = NBD_REPLY_SIZE;
|
||||
|
||||
if ( reply->magic != REPLY_MAGIC ) {
|
||||
debug( "Reply magic is incorrect" );
|
||||
proxy_disconnect_from_upstream( proxy );
|
||||
return 0;
|
||||
}
|
||||
|
||||
debug( "NBD reply received from upstream. Response code: %"PRIu32, reply->error );
|
||||
|
||||
if ( reply->error != 0 ) {
|
||||
warn( "NBD error returned from upstream: %"PRIu32, reply->error );
|
||||
}
|
||||
|
||||
if ( reply->error == 0 && request->type == REQUEST_READ ) {
|
||||
if (readloop( proxy->upstream_fd, rsp_data, request->len ) == -1 ) {
|
||||
debug( "Failed to get reply data from upstream" );
|
||||
proxy_disconnect_from_upstream( proxy );
|
||||
return 0;
|
||||
}
|
||||
rsp_buf_size += request->len;
|
||||
}
|
||||
|
||||
proxy->rsp_buf_size = rsp_buf_size;
|
||||
return rsp_buf_size;
|
||||
}
|
||||
|
||||
/* Write an NBD reply back downstream. Return 0 on failure, 1 on success. */
|
||||
int proxy_send_reply_downstream( struct proxier* proxy )
|
||||
{
|
||||
int result;
|
||||
unsigned char* rsp_buf = proxy->rsp_buf;
|
||||
|
||||
debug(
|
||||
"Writing header (%"PRIu32") + data (%"PRIu32") bytes downstream",
|
||||
NBD_REPLY_SIZE, proxy->rsp_buf_size - NBD_REPLY_SIZE
|
||||
);
|
||||
|
||||
result = writeloop( proxy->downstream_fd, rsp_buf, proxy->rsp_buf_size );
|
||||
if ( result == -1 ) {
|
||||
debug( "Failed to send reply downstream" );
|
||||
return 0;
|
||||
}
|
||||
|
||||
debug( "Reply sent" );
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
/* Here, we negotiate an NBD session with downstream, based on the information
|
||||
* we got on first connection to upstream. Then we wait for a request to come
|
||||
* in from downstream, read it into memory, then send it to upstream. If
|
||||
* upstream dies before responding, we reconnect to upstream and resend it.
|
||||
* Once we've got a response, we write it directly to downstream, and wait for a
|
||||
* new request. When downstream disconnects, or we receive an exit signal (which
|
||||
* can be blocked, unfortunately), we are finished.
|
||||
*
|
||||
* This is the simplest possible nbd proxy I can think of. It may not be at all
|
||||
* performant - let's see.
|
||||
*/
|
||||
|
||||
void proxy_session( struct proxier* proxy )
|
||||
{
|
||||
int downstream_fd = proxy->downstream_fd;
|
||||
uint64_t req_count = 0;
|
||||
int result;
|
||||
|
||||
info( "Beginning proxy session on fd %i", downstream_fd );
|
||||
|
||||
if ( !socket_nbd_write_hello( downstream_fd, proxy->upstream_size ) ) {
|
||||
debug( "Sending hello failed on fd %i, ending session", downstream_fd );
|
||||
return;
|
||||
}
|
||||
|
||||
while( proxy_get_request_from_downstream( proxy ) ) {
|
||||
|
||||
/* Don't start running the request if exit has been signalled */
|
||||
if ( proxy_should_exit( proxy, NULL, 0 ) ) {
|
||||
break;
|
||||
}
|
||||
|
||||
do {
|
||||
result = proxy_run_request_upstream( proxy );
|
||||
} while ( result == 0 );
|
||||
|
||||
/* We have to exit, but don't know if the request was successfully
|
||||
* proxied or not. We could add that knowledge, and attempt to send a
|
||||
* reply downstream if it was, but I don't think it's worth it.
|
||||
*/
|
||||
if ( result == -1 ) {
|
||||
break;
|
||||
}
|
||||
|
||||
if ( !proxy_send_reply_downstream( proxy ) ) {
|
||||
break;
|
||||
}
|
||||
|
||||
proxy->req_buf_size = 0;
|
||||
proxy->rsp_buf_size = 0;
|
||||
|
||||
req_count++;
|
||||
};
|
||||
|
||||
info(
|
||||
"Finished proxy session on fd %i after %"PRIu64" successful request(s)",
|
||||
downstream_fd, req_count
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
/** Accept an NBD socket connection, dispatch appropriately */
|
||||
int proxy_accept( struct proxier* params )
|
||||
{
|
||||
NULLCHECK( params );
|
||||
|
||||
int client_fd;
|
||||
int signal_fd = flexnbd_signal_fd( params->flexnbd );
|
||||
fd_set fds;
|
||||
int should_continue = 1;
|
||||
|
||||
union mysockaddr client_address;
|
||||
socklen_t socklen = sizeof( client_address );
|
||||
|
||||
debug("accept loop starting");
|
||||
|
||||
FD_ZERO(&fds);
|
||||
FD_SET(params->listen_fd, &fds);
|
||||
FD_SET(signal_fd, &fds);
|
||||
|
||||
FATAL_IF_NEGATIVE(
|
||||
sock_try_select(FD_SETSIZE, &fds, NULL, NULL, NULL),
|
||||
SHOW_ERRNO( "select() failed" )
|
||||
);
|
||||
|
||||
if ( proxy_should_exit( params, &fds, 0) ) {
|
||||
should_continue = 0;
|
||||
}
|
||||
|
||||
if ( should_continue && FD_ISSET( params->listen_fd, &fds ) ) {
|
||||
client_fd = accept( params->listen_fd, &client_address.generic, &socklen );
|
||||
|
||||
if ( sock_set_tcp_nodelay(client_fd, 1) == -1 ) {
|
||||
warn( SHOW_ERRNO( "Failed to set TCP_NODELAY" ) );
|
||||
}
|
||||
|
||||
info( "Accepted nbd client socket fd %d", client_fd );
|
||||
params->downstream_fd = client_fd;
|
||||
proxy_session( params );
|
||||
|
||||
if ( close( params->downstream_fd ) == -1 ) {
|
||||
warn( SHOW_ERRNO( "FIXME: close returned" ) );
|
||||
}
|
||||
|
||||
params->downstream_fd = -1;
|
||||
}
|
||||
|
||||
return should_continue;
|
||||
}
|
||||
|
||||
|
||||
void proxy_accept_loop( struct proxier* params )
|
||||
{
|
||||
NULLCHECK( params );
|
||||
while( proxy_accept( params ) );
|
||||
}
|
||||
|
||||
/** Closes sockets, frees memory and waits for all requests to clear */
|
||||
void proxy_cleanup( struct proxier* params )
|
||||
{
|
||||
NULLCHECK( params );
|
||||
|
||||
info( "cleaning up" );
|
||||
|
||||
if ( -1 != params->listen_fd ) {
|
||||
close( params->listen_fd );
|
||||
}
|
||||
|
||||
debug( "Cleanup done" );
|
||||
}
|
||||
|
||||
/** Full lifecycle of the proxier */
|
||||
int do_proxy( struct proxier* params )
|
||||
{
|
||||
NULLCHECK( params );
|
||||
|
||||
error_set_handler( (cleanup_handler*) proxy_cleanup, params );
|
||||
|
||||
debug( "Ensuring upstream server is open" );
|
||||
|
||||
if ( !proxy_connect_to_upstream( params ) ) {
|
||||
info( "Couldn't connect to upstream server during initialization" );
|
||||
proxy_cleanup( params );
|
||||
return 1;
|
||||
};
|
||||
|
||||
proxy_open_listen_socket( params );
|
||||
proxy_accept_loop( params );
|
||||
proxy_cleanup( params );
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
71
src/proxy.h
Normal file
71
src/proxy.h
Normal file
@@ -0,0 +1,71 @@
|
||||
#ifndef PROXY_H
|
||||
#define PROXY_H
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "flexnbd.h"
|
||||
#include "parse.h"
|
||||
#include "nbdtypes.h"
|
||||
#include "self_pipe.h"
|
||||
|
||||
struct proxier {
|
||||
/* The flexnbd wrapper this proxier is attached to */
|
||||
struct flexnbd* flexnbd;
|
||||
|
||||
/** address/port to bind to */
|
||||
union mysockaddr listen_on;
|
||||
|
||||
/** address/port to connect to */
|
||||
union mysockaddr connect_to;
|
||||
|
||||
/** address to bind to when making outgoing connections */
|
||||
union mysockaddr connect_from;
|
||||
|
||||
/* The socket we listen() on and accept() against */
|
||||
int listen_fd;
|
||||
|
||||
/* The socket returned by accept() that we receive requests from and send
|
||||
* responses to
|
||||
*/
|
||||
int downstream_fd;
|
||||
|
||||
/* The socket returned by connect() that we send requests to and receive
|
||||
* responses from
|
||||
*/
|
||||
int upstream_fd;
|
||||
|
||||
/* This is the size we advertise to the downstream server */
|
||||
off64_t upstream_size;
|
||||
|
||||
/* Scratch space for the current NBD request from downstream */
|
||||
unsigned char* req_buf;
|
||||
|
||||
/* Number of bytes currently sat in req_buf */
|
||||
size_t req_buf_size;
|
||||
|
||||
/* We transform the raw request header into here */
|
||||
struct nbd_request req_hdr;
|
||||
|
||||
/* Scratch space for the current NBD reply from upstream */
|
||||
unsigned char* rsp_buf;
|
||||
|
||||
/* Number of bytes currently sat in rsp_buf */
|
||||
size_t rsp_buf_size;
|
||||
|
||||
/* We transform the raw reply header into here */
|
||||
struct nbd_reply rsp_hdr;
|
||||
};
|
||||
|
||||
struct proxier* proxy_create(
|
||||
struct flexnbd * flexnbd,
|
||||
char* s_downstream_address,
|
||||
char* s_downstream_port,
|
||||
char* s_upstream_address,
|
||||
char* s_upstream_port,
|
||||
char* s_upstream_bind );
|
||||
int do_proxy( struct proxier* proxy );
|
||||
void proxy_destroy( struct proxier* proxy );
|
||||
|
||||
#endif
|
||||
|
Reference in New Issue
Block a user