Merge branch 'release-to-master' into 'master'

Release 0.5.0 to master

See merge request open-source/flexnbd-c!60

.gitlab-ci.yml

@@ -2,17 +2,20 @@ stages:
   - package
   - publish
 
-package:jessie: &package
+.package: &package
   stage: package
   image: $CI_REGISTRY/docker-images/layers:$DISTRO-deb
-  variables:
-    DISTRO: jessie
   script:
     - package
   artifacts:
     paths:
       - pkg/
 
+package:jessie:
+  <<: *package
+  variables:
+    DISTRO: jessie
+
 package:stretch:
   <<: *package
   variables:

CONTRIBUTING.md

@@ -0,0 +1,24 @@
+# Contribution guide
+
+The code is formatted using the K&R style of "indent".
+
+```
+indent -kr <files go here>
+```
+
+The C unit tests have also been indented in the same way, but manually adjsted
+such that the functions follow the normal libcheck layout.
+
+```c
+START_TEST( ... ) {
+
+
+}
+END TEST
+```
+
+Indent tends to mangle the `END_TEST` macro, so that will need adjusting if
+`indent` is run over the test files again.
+
+
+

Makefile

@@ -109,7 +109,6 @@ install:
 clean:
 	rm -rf build/*
 
-
 .PHONY: clean objs check_objs all server proxy check_bins check doc build test acceptance
 
 # Include extra dependencies at the end, NOT before 'all'

README.proxy.txt

@@ -169,6 +169,11 @@ That is, the '=' is required. This is a limitation of getopt-long.
 If no cache size is given, a size of 4096 bytes is assumed. Caching can
 be explicitly disabled by setting a size of 0.
 
+ENVIRONMENT
+
+ FLEXNBD_UPSTREAM_TIMEOUT   The timeout in seconds for the proxy communicating
+                            with the upstream server defaults to 30 seconds.
+
 BUGS
 
 Should be reported via GitHub.

debian/changelog

@@ -1,9 +1,53 @@
-flexnbd (0.1.8) UNRELEASED; urgency=medium
+flexnbd (0.5.0) stable; urgency=medium
 
-  * Set TCP keepalive on sockets so broken connections are reaped (#33, !33)
+  [ Patrick J Cherry ]
+  * Explicitly close the server control socket, and wait for it to close, to
+    prevent deadlocks during the server clean-up process (#40 !58)
+  * Ensure mirroring can be restarted after a break command is sent to the
+    source (#37, !59)
+
+ -- James Carter <james.carter@bytemark.co.uk>  Fri, 11 Jan 2019 10:37:23 +0000
+
+flexnbd (0.4.0) stable; urgency=medium
+
+  * Ensure proxy state is completely reset before upstream init is read,
+    ensuring any waiting requests are fully replayed (#39, !54)
+
+ -- Patrick J Cherry <patrick@bytemark.co.uk>  Thu, 15 Nov 2018 14:24:26 +0000
+
+flexnbd (0.3.0) stable; urgency=medium
+
+  * Force a msync after every write, ignoring FUA flag, or lack thereof (!51).
+
+ -- Patrick J Cherry <patrick@bytemark.co.uk>  Tue, 24 Apr 2018 12:05:43 +0100
+
+flexnbd (0.2.0) stable; urgency=medium
+
+  [ James Carter ]
+  * Set TCP keepalive on sockets so broken connections are reaped (#33, !33,
+    !36)
   * Add a context to logs to make debugging problems easier (#34, !34)
 
- -- James Carter <james.carter@bytemark.co.uk>  Thu, 11 Jan 2018 10:05:35 +0000
+  [ Chris Cottam ]
+  * Increased NBD_MAX_SIZE from 1MB to 32MB for qemu 2.11 (!35)
+
+  [ Patrick J Cherry ]
+  * Added FLUSH and FUA support (!38) 
+  * Server returns ENOSPC in response to writes beyond the end of the
+    filesystem, and EINVAL to unknown commands. (#36, !40)
+  * Proxy passes all NBD protocol errors through to the client instead of
+    disconnecting and retrying (#36, !40)
+  * Fix struct types in readwrite.c (#35, !41)
+  * Ignore ends of discs that stray outside of 512-byte sector sizes (!42). 
+  * Tweak logging for readloop failures (!44)
+  * Alter semantics of NBD_MAX_BLOCK_SIZE to remove struct size overheads when
+    calculating if a request exceeds the max block size (!45)
+  * Added tests for setting TCP_NODELAY on upstream-reconnections in the
+    proxy, and refactored the other LD_PRELOAD tests (!43)
+  * Clean up dead threads before calculating the number of connected clients
+    on the status command (!46)
+
+ -- Patrick J Cherry <patrick@bytemark.co.uk>  Tue, 20 Feb 2018 11:43:22 +0000
 
 flexnbd (0.1.7) stable; urgency=medium
 

src/common/ioutil.c

@@ -45,8 +45,7 @@ int build_allocation_map(struct bitset * allocation_map, int fd)
 	if (ioctl(fd, FS_IOC_FIEMAP, fiemap) < 0) {
 	    debug("Couldn't get fiemap, returning no allocation_map");
 	    return 0;		/* it's up to the caller to free the map */
-		}
+	} else {
-		else {
 	    for (unsigned int i = 0; i < fiemap->fm_mapped_extents; i++) {
 		bitset_set_range(allocation_map,
 				 fiemap->fm_extents[i].fe_logical,
@@ -58,12 +57,10 @@ int build_allocation_map(struct bitset * allocation_map, int fd)
 	     * if we've actually hit max_offsets.
 	     */
 	    if (fiemap->fm_mapped_extents > 0) {
-				struct fiemap_extent *last = &fiemap->fm_extents[
+		struct fiemap_extent *last =
-					fiemap->fm_mapped_extents-1
+		    &fiemap->fm_extents[fiemap->fm_mapped_extents - 1];
-				];
 		offset = last->fe_logical + last->fe_length;
-			}
+	    } else {
-			else {
 		offset += fiemap->fm_length;
 	    }
 	}
@@ -74,7 +71,8 @@ int build_allocation_map(struct bitset * allocation_map, int fd)
 }
 
 
-int open_and_mmap(const char* filename, int* out_fd, uint64_t *out_size, void **out_map)
+int open_and_mmap(const char *filename, int *out_fd, uint64_t * out_size,
+		  void **out_map)
 {
     /* 
      * size and out_size are intentionally of different types.
@@ -85,7 +83,7 @@ int open_and_mmap(const char* filename, int* out_fd, uint64_t *out_size, void **
     off64_t size;
 
     /* O_DIRECT should not be used with mmap() */
-	*out_fd = open(filename, O_RDWR | O_SYNC );
+    *out_fd = open(filename, O_RDWR | O_NOATIME);
 
     if (*out_fd < 1) {
 	warn("open(%s) failed: does it exist?", filename);
@@ -97,6 +95,15 @@ int open_and_mmap(const char* filename, int* out_fd, uint64_t *out_size, void **
 	warn("lseek64() failed");
 	return size;
     }
+
+    /* If discs are not in multiples of 512, then odd things happen,
+     * resulting in reads/writes past the ends of files.
+     */
+    if (size != (size & (~0x1ff))) {
+	warn("file does not fit into 512-byte sectors; the end of the file will be ignored.");
+	size &= ~0x1ff;
+    }
+
     if (out_size) {
 	*out_size = size;
     }
@@ -108,9 +115,9 @@ int open_and_mmap(const char* filename, int* out_fd, uint64_t *out_size, void **
 	    warn("mmap64() failed");
 	    return -1;
 	}
-		debug("opened %s size %ld on fd %d @ %p", filename, size, *out_fd, *out_map);
+	debug("opened %s size %ld on fd %d @ %p", filename, size, *out_fd,
-	}
+	      *out_map);
-	else {
+    } else {
 	debug("opened %s size %ld on fd %d", filename, size, *out_fd);
     }
 
@@ -141,7 +148,8 @@ int readloop(int filedes, void *buffer, size_t size)
 	ssize_t result = read(filedes, buffer + readden, size - readden);
 
 	if (result == 0 /* EOF */ ) {
-			warn( "end-of-file detected while reading after %i bytes", readden );
+	    warn("end-of-file detected while reading after %i bytes",
+		 readden);
 	    return -1;
 	}
 
@@ -161,7 +169,9 @@ int sendfileloop(int out_fd, int in_fd, off64_t *offset, size_t count)
     size_t sent = 0;
     while (sent < count) {
 	ssize_t result = sendfile64(out_fd, in_fd, offset, count - sent);
-		debug("sendfile64(out_fd=%d, in_fd=%d, offset=%p, count-sent=%ld) = %ld", out_fd, in_fd, offset, count-sent, result);
+	debug
+	    ("sendfile64(out_fd=%d, in_fd=%d, offset=%p, count-sent=%ld) = %ld",
+	     out_fd, in_fd, offset, count - sent, result);
 
 	if (result == -1) {
 	    debug("%s (%i) calling sendfile64()", strerror(errno), errno);
@@ -175,7 +185,8 @@ int sendfileloop(int out_fd, int in_fd, off64_t *offset, size_t count)
 }
 
 #include <errno.h>
-ssize_t spliceloop(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, size_t len, unsigned int flags2)
+ssize_t spliceloop(int fd_in, loff_t * off_in, int fd_out,
+		   loff_t * off_out, size_t len, unsigned int flags2)
 {
     const unsigned int flags = SPLICE_F_MORE | SPLICE_F_MOVE | flags2;
     size_t spliced = 0;
@@ -183,13 +194,13 @@ ssize_t spliceloop(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, size_
     //debug("spliceloop(%d, %ld, %d, %ld, %ld)", fd_in, off_in ? *off_in : 0, fd_out, off_out ? *off_out : 0, len);
 
     while (spliced < len) {
-		ssize_t result = splice(fd_in, off_in, fd_out, off_out, len, flags);
+	ssize_t result =
+	    splice(fd_in, off_in, fd_out, off_out, len, flags);
 	if (result < 0) {
 	    //debug("result=%ld (%s), spliced=%ld, len=%ld", result, strerror(errno), spliced, len);
 	    if (errno == EAGAIN && (flags & SPLICE_F_NONBLOCK)) {
 		return spliced;
-			}
+	    } else {
-			else {
 		return -1;
 	    }
 	} else {
@@ -213,13 +224,19 @@ int splice_via_pipe_loop(int fd_in, int fd_out, size_t len)
 
     while (spliced < len) {
 	ssize_t run = len - spliced;
-		ssize_t s2, s1 = spliceloop(fd_in, NULL, pipefd[1], NULL, run, SPLICE_F_NONBLOCK);
+	ssize_t s2, s1 =
+	    spliceloop(fd_in, NULL, pipefd[1], NULL, run,
+		       SPLICE_F_NONBLOCK);
 	/*if (run > 65535)
 	   run = 65535; */
-		if (s1 < 0) { break; }
+	if (s1 < 0) {
+	    break;
+	}
 
 	s2 = spliceloop(pipefd[0], NULL, fd_out, NULL, s1, 0);
-		if (s2 < 0) { break; }
+	if (s2 < 0) {
+	    break;
+	}
 	spliced += s2;
     }
     close(pipefd[0]);
@@ -241,7 +258,9 @@ int read_until_newline(int fd, char* buf, int bufsize)
 
     for (cur = 0; cur < bufsize; cur++) {
 	int result = read(fd, buf + cur, 1);
-		if (result <= 0) { return -1; }
+	if (result <= 0) {
+	    return -1;
+	}
 	if (buf[cur] == 10) {
 	    buf[cur] = '\0';
 	    break;
@@ -266,7 +285,9 @@ int read_lines_until_blankline(int fd, int max_line_length, char ***lines)
 	 * -1 for an eof
 	 * -1 for a read error
 	 */
-		if (readden <= 1) { return lines_count; }
+	if (readden <= 1) {
+	    return lines_count;
+	}
 	*lines = xrealloc(*lines, (lines_count + 1) * sizeof(char *));
 	(*lines)[lines_count] = strdup(line);
 	if ((*lines)[lines_count][0] == 0) {
@@ -304,7 +325,8 @@ ssize_t iobuf_read(int fd, struct iobuf *iobuf, size_t default_size )
     }
 
     left = iobuf->size - iobuf->needle;
-	debug( "Reading %"PRIu32" of %"PRIu32" bytes from fd %i", left, iobuf->size, fd );
+    debug("Reading %" PRIu32 " of %" PRIu32 " bytes from fd %i", left,
+	  iobuf->size, fd);
 
     count = read(fd, iobuf->buf + iobuf->needle, left);
 
@@ -332,7 +354,8 @@ ssize_t iobuf_write( int fd, struct iobuf *iobuf )
     size_t left = iobuf->size - iobuf->needle;
     ssize_t count;
 
-	debug( "Writing %"PRIu32" of %"PRIu32" bytes to fd %i", left, iobuf->size, fd );
+    debug("Writing %" PRIu32 " of %" PRIu32 " bytes to fd %i", left,
+	  iobuf->size, fd);
     count = write(fd, iobuf->buf + iobuf->needle, left);
 
     if (count >= 0) {

src/common/ioutil.h

@@ -38,7 +38,8 @@ int readloop(int filedes, void *buffer, size_t size);
 int sendfileloop(int out_fd, int in_fd, off64_t * offset, size_t count);
 
 /** Repeat a splice() operation until we have 'len' bytes. */
-ssize_t spliceloop(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, size_t len, unsigned int flags2);
+ssize_t spliceloop(int fd_in, loff_t * off_in, int fd_out,
+		   loff_t * off_out, size_t len, unsigned int flags2);
 
 /** Copy ''len'' bytes from ''fd_in'' to ''fd_out'' by creating a temporary
   * pipe and using the Linux splice call repeatedly until it has transferred
@@ -65,7 +66,8 @@ int read_lines_until_blankline(int fd, int max_line_length, char ***lines);
   * ''out_size'' and the address of the mmap in ''out_map''.  If anything goes
   * wrong, returns -1 setting errno, otherwise 0.
   */
-int open_and_mmap( const char* filename, int* out_fd, uint64_t* out_size, void **out_map);
+int open_and_mmap(const char *filename, int *out_fd, uint64_t * out_size,
+		  void **out_map);
 
 
 /** Check to see whether the given file descriptor is closed.
@@ -73,4 +75,3 @@ int open_and_mmap( const char* filename, int* out_fd, uint64_t* out_size, void *
 int fd_is_closed(int fd_in);
 
 #endif
-

src/common/mode.h

@@ -94,4 +94,3 @@ void mode(char* mode, int argc, char **argv);
 char *help_help_text;
 
 #endif
-

src/common/nbdtypes.c

@@ -13,6 +13,7 @@ void nbd_r2h_init( struct nbd_init_raw * from, struct nbd_init * to )
     memcpy(to->passwd, from->passwd, 8);
     to->magic = be64toh(from->magic);
     to->size = be64toh(from->size);
+    to->flags = be32toh(from->flags);
 }
 
 void nbd_h2r_init(struct nbd_init *from, struct nbd_init_raw *to)
@@ -20,39 +21,41 @@ void nbd_h2r_init( struct nbd_init * from, struct nbd_init_raw * to)
     memcpy(to->passwd, from->passwd, 8);
     to->magic = htobe64(from->magic);
     to->size = htobe64(from->size);
+    to->flags = htobe32(from->flags);
 }
 
 
 void nbd_r2h_request(struct nbd_request_raw *from, struct nbd_request *to)
-{
-	to->magic = htobe32( from->magic );
-	to->type = htobe32( from->type );
-	to->handle.w = from->handle.w;
-	to->from = htobe64( from->from );
-	to->len = htobe32( from->len );
-}
-
-void nbd_h2r_request( struct nbd_request * from, struct nbd_request_raw * to )
 {
     to->magic = be32toh(from->magic);
-	to->type = be32toh( from->type );
+    to->flags = be16toh(from->flags);
+    to->type = be16toh(from->type);
     to->handle.w = from->handle.w;
     to->from = be64toh(from->from);
     to->len = be32toh(from->len);
 }
 
-
+void nbd_h2r_request(struct nbd_request *from, struct nbd_request_raw *to)
-void nbd_r2h_reply( struct nbd_reply_raw * from, struct nbd_reply * to )
 {
     to->magic = htobe32(from->magic);
-	to->error = htobe32( from->error );
+    to->flags = htobe16(from->flags);
+    to->type = htobe16(from->type);
     to->handle.w = from->handle.w;
+    to->from = htobe64(from->from);
+    to->len = htobe32(from->len);
 }
 
-void nbd_h2r_reply( struct nbd_reply * from, struct nbd_reply_raw * to )
+
+void nbd_r2h_reply(struct nbd_reply_raw *from, struct nbd_reply *to)
 {
     to->magic = be32toh(from->magic);
     to->error = be32toh(from->error);
     to->handle.w = from->handle.w;
 }
 
+void nbd_h2r_reply(struct nbd_reply *from, struct nbd_reply_raw *to)
+{
+    to->magic = htobe32(from->magic);
+    to->error = htobe32(from->error);
+    to->handle.w = from->handle.w;
+}

src/common/nbdtypes.h

@@ -7,15 +7,38 @@
 #define INIT_MAGIC 0x0000420281861253
 #define REQUEST_MAGIC 0x25609513
 #define REPLY_MAGIC 0x67446698
+
 #define REQUEST_READ 0
 #define REQUEST_WRITE 1
 #define REQUEST_DISCONNECT 2
+#define REQUEST_FLUSH 3
 
-/* The top 2 bytes of the type field are overloaded and can contain flags */
+/* values for transmission flag field */
-#define REQUEST_MASK 0x0000ffff
+#define FLAG_HAS_FLAGS	(1 << 0)	/* Flags are there */
+#define FLAG_SEND_FLUSH	(1 << 2)	/* Send FLUSH */
+#define FLAG_SEND_FUA	(1 << 3)	/* Send FUA (Force Unit Access) */
+
+/* values for command flag field */
+#define CMD_FLAG_FUA     (1 << 0)
+
+#if 0
+/* Not yet implemented by flexnbd */
+#define REQUEST_TRIM 4
+#define REQUEST_WRITE_ZEROES 6
+
+#define FLAG_READ_ONLY	(1 << 1)	/* Device is read-only */
+#define FLAG_ROTATIONAL	(1 << 4)	/* Use elevator algorithm - rotational media */
+#define FLAG_SEND_TRIM	(1 << 5)	/* Send TRIM (discard) */
+#define FLAG_SEND_WRITE_ZEROES (1 << 6)	/* Send NBD_CMD_WRITE_ZEROES */
+#define FLAG_CAN_MULTI_CONN    (1 << 8)	/* multiple connections are okay */
+
+#define CMD_FLAG_NO_HOLE (1 << 1)
+#endif
 
 
-/* 1MiB is the de-facto standard for maximum size of header + data */
+/* 32 MiB is the maximum qemu will send you:
+ * https://github.com/qemu/qemu/blob/v2.11.0/include/block/nbd.h#L183
+ */
 #define NBD_MAX_SIZE ( 32 * 1024 * 1024 )
 
 #define NBD_REQUEST_SIZE ( sizeof( struct nbd_request_raw ) )
@@ -38,12 +61,14 @@ struct nbd_init_raw {
     char passwd[8];
     __be64 magic;
     __be64 size;
-	char reserved[128];
+    __be32 flags;
+    char reserved[124];
 };
 
 struct nbd_request_raw {
     __be32 magic;
-	__be32 type;    /* == READ || == WRITE  */
+    __be16 flags;
+    __be16 type;		/* == READ || == WRITE || == FLUSH */
     nbd_handle_t handle;
     __be64 from;
     __be32 len;
@@ -55,18 +80,18 @@ struct nbd_reply_raw {
     nbd_handle_t handle;	/* handle you got from request  */
 };
 
-
-
 struct nbd_init {
     char passwd[8];
     uint64_t magic;
     uint64_t size;
-	char reserved[128];
+    uint32_t flags;
+    char reserved[124];
 };
 
 struct nbd_request {
     uint32_t magic;
-	uint32_t type;    /* == READ || == WRITE || == DISCONNECT */
+    uint16_t flags;
+    uint16_t type;		/* == READ || == WRITE || == DISCONNECT || == FLUSH */
     nbd_handle_t handle;
     uint64_t from;
     uint32_t len;
@@ -87,4 +112,3 @@ void nbd_h2r_request( struct nbd_request * from, struct nbd_request_raw * to );
 void nbd_h2r_reply(struct nbd_reply *from, struct nbd_reply_raw *to);
 
 #endif
-

src/common/parse.c

@@ -22,7 +22,9 @@ int parse_ip_to_sockaddr(struct sockaddr* out, char* src)
     /* allow user to start with [ and end with any other invalid char */
     {
 	int i = 0, j = 0;
-		if (src[i] == '[') { i++; }
+	if (src[i] == '[') {
+	    i++;
+	}
 	for (; i < 64 && IS_IP_VALID_CHAR(src[i]); i++) {
 	    temp[j++] = src[i];
 	}
@@ -72,8 +74,7 @@ int parse_acl(struct ip_and_mask (**out)[], int max, char **entries)
     if (max == 0) {
 	*out = NULL;
 	return 0;
-	}
+    } else {
-	else {
 	list = xmalloc(max * sizeof(struct ip_and_mask));
 	*out = (struct ip_and_mask(*)[]) list;
 	debug("acl alloc: %p", *out);
@@ -88,16 +89,14 @@ int parse_acl(struct ip_and_mask (**out)[], int max, char **entries)
 	    return i;
 	}
 
-		for (j=0; entries[i][j] && entries[i][j] != '/'; j++)
+	for (j = 0; entries[i][j] && entries[i][j] != '/'; j++);	// increment j!
-			; // increment j!
 
 	if (entries[i][j] == '/') {
 	    outentry->mask = atoi(entries[i] + j + 1);
 	    if (outentry->mask < 1 || outentry->mask > MAX_MASK_BITS) {
 		return i;
 	    }
-		}
+	} else {
-		else {
 	    outentry->mask = MAX_MASK_BITS;
 	}
 #		undef MAX_MASK_BITS
@@ -124,4 +123,3 @@ void parse_port( char *s_port, struct sockaddr_in *out )
     }
     out->sin_port = htobe16(raw_port);
 }
-

src/common/parse.h

@@ -26,4 +26,3 @@ int parse_acl(struct ip_and_mask (**out)[], int max, char **entries);
 void parse_port(char *s_port, struct sockaddr_in *out);
 
 #endif
-

src/common/readwrite.c

@@ -10,7 +10,9 @@
 
 int socket_connect(struct sockaddr *to, struct sockaddr *from)
 {
-	int fd = socket(to->sa_family == AF_INET ? PF_INET : PF_INET6, SOCK_STREAM, 0);
+    int fd =
+	socket(to->sa_family == AF_INET ? PF_INET : PF_INET6, SOCK_STREAM,
+	       0);
     if (fd < 0) {
 	warn("Couldn't create client socket");
 	return -1;
@@ -41,7 +43,8 @@ int socket_connect(struct sockaddr* to, struct sockaddr* from)
     return fd;
 }
 
-int nbd_check_hello( struct nbd_init_raw* init_raw, uint64_t* out_size )
+int nbd_check_hello(struct nbd_init_raw *init_raw, uint64_t * out_size,
+		    uint32_t * out_flags)
 {
     if (strncmp(init_raw->passwd, INIT_PASSWD, 8) != 0) {
 	warn("wrong passwd");
@@ -56,13 +59,18 @@ int nbd_check_hello( struct nbd_init_raw* init_raw, uint64_t* out_size )
 	*out_size = be64toh(init_raw->size);
     }
 
+    if (NULL != out_flags) {
+	*out_flags = be32toh(init_raw->flags);
+    }
+
     return 1;
   fail:
     return 0;
 
 }
 
-int socket_nbd_read_hello( int fd, uint64_t* out_size )
+int socket_nbd_read_hello(int fd, uint64_t * out_size,
+			  uint32_t * out_flags)
 {
     struct nbd_init_raw init_raw;
 
@@ -72,16 +80,18 @@ int socket_nbd_read_hello( int fd, uint64_t* out_size )
 	return 0;
     }
 
-	return nbd_check_hello( &init_raw, out_size );
+    return nbd_check_hello(&init_raw, out_size, out_flags);
 }
 
-void nbd_hello_to_buf( struct nbd_init_raw *buf, off64_t out_size )
+void nbd_hello_to_buf(struct nbd_init_raw *buf, off64_t out_size,
+		      uint32_t out_flags)
 {
     struct nbd_init init;
 
     memcpy(&init.passwd, INIT_PASSWD, 8);
     init.magic = INIT_MAGIC;
     init.size = out_size;
+    init.flags = out_flags;
 
     memset(buf, 0, sizeof(struct nbd_init_raw));	// ensure reserved is 0s
     nbd_h2r_init(&init, buf);
@@ -89,10 +99,10 @@ void nbd_hello_to_buf( struct nbd_init_raw *buf, off64_t out_size )
     return;
 }
 
-int socket_nbd_write_hello(int fd, off64_t out_size)
+int socket_nbd_write_hello(int fd, off64_t out_size, uint32_t out_flags)
 {
     struct nbd_init_raw init_raw;
-	nbd_hello_to_buf( &init_raw, out_size );
+    nbd_hello_to_buf(&init_raw, out_size, out_flags);
 
     if (0 > writeloop(fd, &init_raw, sizeof(init_raw))) {
 	warn(SHOW_ERRNO("failed to write hello to socket"));
@@ -101,20 +111,25 @@ int socket_nbd_write_hello(int fd, off64_t out_size)
     return 1;
 }
 
-void fill_request(struct nbd_request *request, int type, uint64_t from, uint32_t len)
+void fill_request(struct nbd_request_raw *request_raw, uint16_t type,
+		  uint16_t flags, uint64_t from, uint32_t len)
 {
-	request->magic  = htobe32(REQUEST_MAGIC);
+    request_raw->magic = htobe32(REQUEST_MAGIC);
-	request->type   = htobe32(type);
+    request_raw->type = htobe16(type);
-	request->handle.w = (((uint64_t)rand()) << 32) | ((uint64_t)rand());
+    request_raw->flags = htobe16(flags);
-	request->from   = htobe64(from);
+    request_raw->handle.w =
-	request->len    = htobe32(len);
+	(((uint64_t) rand()) << 32) | ((uint64_t) rand());
+    request_raw->from = htobe64(from);
+    request_raw->len = htobe32(len);
 }
 
-void read_reply(int fd, struct nbd_request *request, struct nbd_reply *reply)
+void read_reply(int fd, uint64_t request_raw_handle,
+		struct nbd_reply *reply)
 {
     struct nbd_reply_raw reply_raw;
 
-	ERROR_IF_NEGATIVE(readloop(fd, &reply_raw, sizeof(struct nbd_reply_raw)),
+    ERROR_IF_NEGATIVE(readloop
+		      (fd, &reply_raw, sizeof(struct nbd_reply_raw)),
 		      "Couldn't read reply");
 
     nbd_r2h_reply(&reply_raw, reply);
@@ -125,7 +140,7 @@ void read_reply(int fd, struct nbd_request *request, struct nbd_reply *reply)
     if (reply->error != 0) {
 	error("Server replied with error %d", reply->error);
     }
-	if (request->handle.w != reply->handle.w) {
+    if (request_raw_handle != reply_raw.handle.w) {
 	error("Did not reply with correct handle");
     }
 }
@@ -139,81 +154,77 @@ void wait_for_data( int fd, int timeout_secs )
     FD_ZERO(&fds);
     FD_SET(fd, &fds);
 
-	selected = sock_try_select(
+    selected =
-		FD_SETSIZE, &fds, NULL, NULL, timeout_secs >=0 ? &tv : NULL
+	sock_try_select(FD_SETSIZE, &fds, NULL, NULL,
-	);
+			timeout_secs >= 0 ? &tv : NULL);
 
     FATAL_IF(-1 == selected, "Select failed");
     ERROR_IF(0 == selected, "Timed out waiting for reply");
 }
 
 
-void socket_nbd_read(int fd, uint64_t from, uint32_t len, int out_fd, void* out_buf, int timeout_secs)
+void socket_nbd_read(int fd, uint64_t from, uint32_t len, int out_fd,
+		     void *out_buf, int timeout_secs)
 {
-	struct nbd_request request;
+    struct nbd_request_raw request_raw;
     struct nbd_reply reply;
 
-	fill_request(&request, REQUEST_READ, from, len);
+    fill_request(&request_raw, REQUEST_READ, 0, from, len);
-	FATAL_IF_NEGATIVE(writeloop(fd, &request, sizeof(request)),
+    FATAL_IF_NEGATIVE(writeloop(fd, &request_raw, sizeof(request_raw)),
 		      "Couldn't write request");
 
+
     wait_for_data(fd, timeout_secs);
-	read_reply(fd, &request, &reply);
+    read_reply(fd, request_raw.handle.w, &reply);
 
     if (out_buf) {
-		FATAL_IF_NEGATIVE(readloop(fd, out_buf, len),
+	FATAL_IF_NEGATIVE(readloop(fd, out_buf, len), "Read failed");
-		  "Read failed");
+    } else {
-	}
+	FATAL_IF_NEGATIVE(splice_via_pipe_loop(fd, out_fd, len),
-	else {
+			  "Splice failed");
-		FATAL_IF_NEGATIVE(
-			splice_via_pipe_loop(fd, out_fd, len),
-			"Splice failed"
-		);
     }
 }
 
-void socket_nbd_write(int fd, uint64_t from, uint32_t len, int in_fd, void* in_buf, int timeout_secs)
+void socket_nbd_write(int fd, uint64_t from, uint32_t len, int in_fd,
+		      void *in_buf, int timeout_secs)
 {
-	struct nbd_request request;
+    struct nbd_request_raw request_raw;
     struct nbd_reply reply;
 
-	fill_request(&request, REQUEST_WRITE, from, len);
+    fill_request(&request_raw, REQUEST_WRITE, 0, from, len);
-	ERROR_IF_NEGATIVE(writeloop(fd, &request, sizeof(request)),
+    ERROR_IF_NEGATIVE(writeloop(fd, &request_raw, sizeof(request_raw)),
 		      "Couldn't write request");
 
     if (in_buf) {
-		ERROR_IF_NEGATIVE(writeloop(fd, in_buf, len),
+	ERROR_IF_NEGATIVE(writeloop(fd, in_buf, len), "Write failed");
-		  "Write failed");
+    } else {
-	}
+	ERROR_IF_NEGATIVE(splice_via_pipe_loop(in_fd, fd, len),
-	else {
+			  "Splice failed");
-		ERROR_IF_NEGATIVE(
-			splice_via_pipe_loop(in_fd, fd, len),
-			"Splice failed"
-		);
     }
 
     wait_for_data(fd, timeout_secs);
-	read_reply(fd, &request, &reply);
+    read_reply(fd, request_raw.handle.w, &reply);
 }
 
 
 int socket_nbd_disconnect(int fd)
 {
     int success = 1;
-	struct nbd_request request;
+    struct nbd_request_raw request_raw;
 
-	fill_request( &request, REQUEST_DISCONNECT, 0, 0 );
+    fill_request(&request_raw, REQUEST_DISCONNECT, 0, 0, 0);
     /* FIXME: This shouldn't be a FATAL error.  We should just drop
      * the mirror without affecting the main server.
      */
-	FATAL_IF_NEGATIVE( writeloop( fd, &request, sizeof( request ) ),
+    FATAL_IF_NEGATIVE(writeloop(fd, &request_raw, sizeof(request_raw)),
 		      "Failed to write the disconnect request.");
     return success;
 }
 
 #define CHECK_RANGE(error_type) { \
 	uint64_t size;\
-	int success = socket_nbd_read_hello(params->client, &size); \
+	uint32_t flags;\
+	int success = socket_nbd_read_hello(params->client, &size, &flags); \
 	if ( success ) {\
 		uint64_t endpoint = params->from + params->len; \
 		if (endpoint > size || \
@@ -231,7 +242,9 @@ int socket_nbd_disconnect( int fd )
 
 void do_read(struct mode_readwrite_params *params)
 {
-	params->client = socket_connect(&params->connect_to.generic, &params->connect_from.generic);
+    params->client =
+	socket_connect(&params->connect_to.generic,
+		       &params->connect_from.generic);
     FATAL_IF_NEGATIVE(params->client, "Couldn't connect.");
     CHECK_RANGE("read");
     socket_nbd_read(params->client, params->from, params->len,
@@ -241,11 +254,12 @@ void do_read(struct mode_readwrite_params* params)
 
 void do_write(struct mode_readwrite_params *params)
 {
-	params->client = socket_connect(&params->connect_to.generic, &params->connect_from.generic);
+    params->client =
+	socket_connect(&params->connect_to.generic,
+		       &params->connect_from.generic);
     FATAL_IF_NEGATIVE(params->client, "Couldn't connect.");
     CHECK_RANGE("write");
     socket_nbd_write(params->client, params->from, params->len,
 		     params->data_fd, NULL, 10);
     close(params->client);
 }
-

src/common/readwrite.h

@@ -7,17 +7,20 @@
 #include "nbdtypes.h"
 
 int socket_connect(struct sockaddr *to, struct sockaddr *from);
-int socket_nbd_read_hello(int fd, uint64_t* size);
+int socket_nbd_read_hello(int fd, uint64_t * size, uint32_t * flags);
-int socket_nbd_write_hello(int fd, uint64_t size);
+int socket_nbd_write_hello(int fd, uint64_t size, uint32_t flags);
-void socket_nbd_read(int fd, uint64_t from, uint32_t len, int out_fd, void* out_buf, int timeout_secs);
+void socket_nbd_read(int fd, uint64_t from, uint32_t len, int out_fd,
-void socket_nbd_write(int fd, uint64_t from, uint32_t len, int out_fd, void* out_buf, int timeout_secs);
+		     void *out_buf, int timeout_secs);
+void socket_nbd_write(int fd, uint64_t from, uint32_t len, int out_fd,
+		      void *out_buf, int timeout_secs);
 int socket_nbd_disconnect(int fd);
 
 /* as you can see, we're slowly accumulating code that should really be in an
  * NBD library */
 
-void nbd_hello_to_buf( struct nbd_init_raw* buf, uint64_t out_size );
+void nbd_hello_to_buf(struct nbd_init_raw *buf, uint64_t out_size,
-int nbd_check_hello( struct nbd_init_raw* init_raw, uint64_t* out_size );
+		      uint32_t out_flags);
+int nbd_check_hello(struct nbd_init_raw *init_raw, uint64_t * out_size,
+		    uint32_t * out_flags);
 
 #endif
-

src/common/remote.c

@@ -24,7 +24,8 @@ void print_response( const char * response )
     fprintf(out, "%s\n", response_text + 2);
 }
 
-void do_remote_command(char* command, char* socket_name, int argc, char** argv)
+void do_remote_command(char *command, char *socket_name, int argc,
+		       char **argv)
 {
     char newline = 10;
     int i;
@@ -40,10 +41,10 @@ void do_remote_command(char* command, char* socket_name, int argc, char** argv)
     address.sun_family = AF_UNIX;
     strncpy(address.sun_path, socket_name, sizeof(address.sun_path));
 
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(connect
-		connect(remote, (struct sockaddr*) &address, sizeof(address)),
+		      (remote, (struct sockaddr *) &address,
-		"Couldn't connect to %s", socket_name
+		       sizeof(address)), "Couldn't connect to %s",
-	);
+		      socket_name);
 
     write(remote, command, strlen(command));
     write(remote, &newline, 1);
@@ -55,13 +56,10 @@ void do_remote_command(char* command, char* socket_name, int argc, char** argv)
     }
     write(remote, &newline, 1);
 
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(read_until_newline(remote, response, max_response),
-		read_until_newline(remote, response, max_response),
+		      "Couldn't read response from %s", socket_name);
-		"Couldn't read response from %s", socket_name
-	);
 
     print_response(response);
 
     exit(atoi(response));
 }
-

src/common/self_pipe.c

@@ -52,7 +52,9 @@ struct self_pipe * self_pipe_create(void)
     struct self_pipe *sig = xmalloc(sizeof(struct self_pipe));
     int fds[2];
 
-	if ( NULL == sig ) { return NULL; }
+    if (NULL == sig) {
+	return NULL;
+    }
 
     if (pipe(fds)) {
 	free(sig);
@@ -60,7 +62,8 @@ struct self_pipe * self_pipe_create(void)
 	return NULL;
     }
 
-	if ( fcntl( fds[0], F_SETFL, O_NONBLOCK ) || fcntl( fds[1], F_SETFL, O_NONBLOCK ) ) {
+    if (fcntl(fds[0], F_SETFL, O_NONBLOCK)
+	|| fcntl(fds[1], F_SETFL, O_NONBLOCK)) {
 	int fcntl_err = errno;
 	while (close(fds[0]) == -1 && errno == EINTR);
 	while (close(fds[1]) == -1 && errno == EINTR);

src/common/sockutil.c

@@ -29,7 +29,8 @@ size_t sockaddr_size( const struct sockaddr* sa )
     return ret;
 }
 
-const char* sockaddr_address_string( const struct sockaddr* sa, char* dest, size_t len )
+const char *sockaddr_address_string(const struct sockaddr *sa, char *dest,
+				    size_t len)
 {
     NULLCHECK(sa);
     NULLCHECK(dest);
@@ -65,7 +66,8 @@ const char* sockaddr_address_string( const struct sockaddr* sa, char* dest, size
 
 int sock_set_reuseaddr(int fd, int optval)
 {
-	return setsockopt( fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval) );
+    return setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &optval,
+		      sizeof(optval));
 }
 
 int sock_set_keepalive_params(int fd, int time, int intvl, int probes)
@@ -81,28 +83,33 @@ int sock_set_keepalive_params( int fd, int time, int intvl, int probes)
 
 int sock_set_keepalive(int fd, int optval)
 {
-	return setsockopt( fd, SOL_SOCKET, SO_KEEPALIVE, &optval, sizeof(optval) );
+    return setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &optval,
+		      sizeof(optval));
 }
 
 int sock_set_tcp_keepidle(int fd, int optval)
 {
-	return setsockopt( fd, IPPROTO_TCP, TCP_KEEPIDLE, &optval, sizeof(optval) );
+    return setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &optval,
+		      sizeof(optval));
 }
 
 int sock_set_tcp_keepintvl(int fd, int optval)
 {
-	return setsockopt( fd, IPPROTO_TCP, TCP_KEEPINTVL, &optval, sizeof(optval) );
+    return setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &optval,
+		      sizeof(optval));
 }
 
 int sock_set_tcp_keepcnt(int fd, int optval)
 {
-	return setsockopt( fd, IPPROTO_TCP, TCP_KEEPCNT, &optval, sizeof(optval) );
+    return setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &optval,
+		      sizeof(optval));
 }
 
 /* Set the tcp_nodelay option */
 int sock_set_tcp_nodelay(int fd, int optval)
 {
-	return setsockopt( fd, IPPROTO_TCP, TCP_NODELAY, &optval, sizeof(optval) );
+    return setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &optval,
+		      sizeof(optval));
 }
 
 int sock_set_tcp_cork(int fd, int optval)
@@ -140,8 +147,7 @@ int sock_try_bind( int fd, const struct sockaddr* sa )
 	if (0 == bind_result) {
 	    info("Bound to %s", s_address);
 	    break;
-		}
+	} else {
-		else {
 	    warn(SHOW_ERRNO("Couldn't bind to %s", s_address));
 
 	    switch (errno) {
@@ -177,7 +183,8 @@ int sock_try_bind( int fd, const struct sockaddr* sa )
     return bind_result;
 }
 
-int sock_try_select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout)
+int sock_try_select(int nfds, fd_set * readfds, fd_set * writefds,
+		    fd_set * exceptfds, struct timeval *timeout)
 {
     int result;
 
@@ -192,14 +199,16 @@ int sock_try_select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptf
     return result;
 }
 
-int sock_try_connect( int fd, struct sockaddr* to, socklen_t addrlen, int wait )
+int sock_try_connect(int fd, struct sockaddr *to, socklen_t addrlen,
+		     int wait)
 {
     fd_set fds;
     struct timeval tv = { wait, 0 };
     int result = 0;
 
     if (sock_set_nonblock(fd, 1) == -1) {
-		warn( SHOW_ERRNO( "Failed to set socket non-blocking for connect()" ) );
+	warn(SHOW_ERRNO
+	     ("Failed to set socket non-blocking for connect()"));
 	return connect(fd, to, addrlen);
     }
 
@@ -284,4 +293,3 @@ int sock_try_close( int fd )
 
     return result;
 }
-

src/common/sockutil.h

@@ -12,7 +12,8 @@ size_t sockaddr_size(const struct sockaddr* sa);
 /* Convert a sockaddr into an address. Like inet_ntop, it returns dest if
  * successful, NULL otherwise. In the latter case, dest will contain "???"
  */
-const char* sockaddr_address_string(const struct sockaddr* sa, char* dest, size_t len);
+const char *sockaddr_address_string(const struct sockaddr *sa, char *dest,
+				    size_t len);
 
 /* Configure TCP keepalive on a socket */
 int sock_set_keepalive_params(int fd, int time, int intvl, int probes);
@@ -44,13 +45,14 @@ int sock_set_nonblock(int fd, int optval);
 int sock_try_bind(int fd, const struct sockaddr *sa);
 
 /* Try to call select(), retrying EINTR */
-int sock_try_select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout);
+int sock_try_select(int nfds, fd_set * readfds, fd_set * writefds,
+		    fd_set * exceptfds, struct timeval *timeout);
 
 /* Try to call connect(), timing out after wait seconds */
-int sock_try_connect( int fd, struct sockaddr* to, socklen_t addrlen, int wait );
+int sock_try_connect(int fd, struct sockaddr *to, socklen_t addrlen,
+		     int wait);
 
 /* Try to call close(), retrying EINTR */
 int sock_try_close(int fd);
 
 #endif
-

src/common/util.c

@@ -26,10 +26,12 @@ void error_handler(int fatal)
 
     if (context) {
 	longjmp(context->jmp, fatal ? 1 : 2);
+    } else {
+	if (fatal) {
+	    abort();
+	} else {
+	    pthread_exit((void *) 1);
 	}
-	else {
-		if ( fatal ) { abort(); }
-		else { pthread_exit((void*) 1); }
     }
 }
 
@@ -43,7 +45,9 @@ void exit_err( const char *msg )
 
 void mylog(int line_level, const char *format, ...)
 {
-	if (line_level < log_level) { return; }
+    if (line_level < log_level) {
+	return;
+    }
 
     va_list argptr;
 
@@ -57,8 +61,7 @@ uint64_t monotonic_time_ms()
     struct timespec ts;
     uint64_t seconds_ms, nanoseconds_ms;
 
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(clock_gettime(CLOCK_MONOTONIC, &ts),
-		clock_gettime(CLOCK_MONOTONIC, &ts),
 		      SHOW_ERRNO("clock_gettime failed")
 	);
 
@@ -75,7 +78,8 @@ uint64_t monotonic_time_ms()
 void *xrealloc(void *ptr, size_t size)
 {
     void *p = realloc(ptr, size);
-	FATAL_IF_NULL(p, "couldn't xrealloc %d bytes", ptr ? "realloc" : "malloc", size);
+    FATAL_IF_NULL(p, "couldn't xrealloc %d bytes",
+		  ptr ? "realloc" : "malloc", size);
     return p;
 }
 
@@ -85,4 +89,3 @@ void* xmalloc(size_t size)
     memset(p, 0, size);
     return p;
 }
-

src/common/util.h

@@ -13,7 +13,8 @@
 void *xrealloc(void *ptr, size_t size);
 void *xmalloc(size_t size);
 
-typedef void (cleanup_handler)(void* /* context */, int /* is fatal? */);
+typedef void (cleanup_handler) (void * /* context */ ,
+				int /* is fatal? */ );
 
 /* set from 0 - 5 depending on what level of verbosity you want */
 extern int log_level;
@@ -162,4 +163,3 @@ uint64_t monotonic_time_ms(void);
 #define WARN_IF_NEGATIVE( value, msg, ... ) if ( value < 0 ) { warn( msg, ##__VA_ARGS__ ); }
 
 #endif
-

src/main.c

@@ -19,4 +19,3 @@ int main(int argc, char** argv)
 
     return 0;
 }
-

src/proxy-main.c

@@ -19,6 +19,7 @@ static struct option proxy_options[] = {
     GETOPT_VERBOSE,
     {0}
 };
+
 static char proxy_short_options[] = "hl:p:C:P:b:" SOPT_QUIET SOPT_VERBOSE;
 static char proxy_help_text[] =
     "Usage: flexnbd-proxy <options>\n\n"
@@ -26,24 +27,24 @@ static char proxy_help_text[] =
     "We can listen on TCP or UNIX socket, but only connect to TCP servers.\n\n"
     HELP_LINE
     "\t--" OPT_ADDR ",-l <ADDR>\tThe address we will bind to as a proxy.\n"
-	"\t--" OPT_PORT ",-p <PORT>\tThe port we will bind to as a proxy, if required.\n"
+    "\t--" OPT_PORT
+    ",-p <PORT>\tThe port we will bind to as a proxy, if required.\n"
     "\t--" OPT_CONNECT_ADDR ",-C <ADDR>\tAddress of the proxied server.\n"
     "\t--" OPT_CONNECT_PORT ",-P <PORT>\tPort of the proxied server.\n"
-	"\t--" OPT_BIND ",-b <ADDR>\tThe address we connect from, as a proxy.\n"
+    "\t--" OPT_BIND
-	"\t--" OPT_CACHE ",-c[=<CACHE-BYTES>]\tUse a RAM read cache of the given size.\n"
+    ",-b <ADDR>\tThe address we connect from, as a proxy.\n" "\t--"
-	QUIET_LINE
+    OPT_CACHE
-	VERBOSE_LINE;
+    ",-c[=<CACHE-BYTES>]\tUse a RAM read cache of the given size.\n"
+    QUIET_LINE VERBOSE_LINE;
 
 static char proxy_default_cache_size[] = "4096";
 
-void read_proxy_param(
+void read_proxy_param(int c,
-			int c,
 		      char **downstream_addr,
 		      char **downstream_port,
 		      char **upstream_addr,
 		      char **upstream_port,
-			char **bind_addr,
+		      char **bind_addr, char **cache_bytes)
-			char **cache_bytes)
 {
     switch (c) {
     case 'h':
@@ -116,34 +117,31 @@ int main( int argc, char *argv[] )
     srand(time(NULL));
 
     while (1) {
-		c = getopt_long( argc, argv, proxy_short_options, proxy_options, NULL );
+	c = getopt_long(argc, argv, proxy_short_options, proxy_options,
-		if ( -1 == c ) { break; }
+			NULL);
+	if (-1 == c) {
+	    break;
+	}
 	read_proxy_param(c,
 			 &downstream_addr,
 			 &downstream_port,
 			 &upstream_addr,
-				&upstream_port,
+			 &upstream_port, &bind_addr, &cache_bytes);
-				&bind_addr,
-				&cache_bytes
-		);
     }
 
     if (NULL == downstream_addr) {
 	fprintf(stderr, "--addr is required.\n");
 	exit_err(proxy_help_text);
     } else if (NULL == upstream_addr || NULL == upstream_port) {
-		fprintf( stderr, "both --conn-addr and --conn-port are required.\n" );
+	fprintf(stderr,
+		"both --conn-addr and --conn-port are required.\n");
 	exit_err(proxy_help_text);
     }
 
-	proxy = proxy_create(
+    proxy = proxy_create(downstream_addr,
-		downstream_addr,
 			 downstream_port,
 			 upstream_addr,
-		upstream_port,
+			 upstream_port, bind_addr, cache_bytes);
-		bind_addr,
-		cache_bytes
-	);
 
     /* Set these *after* proxy has been assigned to */
     sigaction(SIGTERM, &exit_action, NULL);
@@ -152,15 +150,12 @@ int main( int argc, char *argv[] )
     signal(SIGPIPE, SIG_IGN);	/* calls to splice() unhelpfully throw this */
 
     if (NULL != downstream_port) {
-		info(
+	info("Proxying between %s %s (downstream) and %s %s (upstream)",
-			"Proxying between %s %s (downstream) and %s %s (upstream)",
+	     downstream_addr, downstream_port, upstream_addr,
-			downstream_addr, downstream_port, upstream_addr, upstream_port
+	     upstream_port);
-		);
     } else {
-		info(
+	info("Proxying between %s (downstream) and %s %s (upstream)",
-			"Proxying between %s (downstream) and %s %s (upstream)",
+	     downstream_addr, upstream_addr, upstream_port);
-			downstream_addr, upstream_addr, upstream_port
-		);
     }
 
     success = do_proxy(proxy);
@@ -168,4 +163,3 @@ int main( int argc, char *argv[] )
 
     return success ? 0 : 1;
 }
-

src/proxy/prefetch.c

@@ -2,7 +2,8 @@
 #include "util.h"
 
 
-struct prefetch* prefetch_create( size_t size_bytes ){
+struct prefetch *prefetch_create(size_t size_bytes)
+{
 
     struct prefetch *out = xmalloc(sizeof(struct prefetch));
     NULLCHECK(out);
@@ -19,14 +20,16 @@ struct prefetch* prefetch_create( size_t size_bytes ){
 
 }
 
-void prefetch_destroy( struct prefetch *prefetch ) {
+void prefetch_destroy(struct prefetch *prefetch)
+{
     if (prefetch) {
 	free(prefetch->buffer);
 	free(prefetch);
     }
 }
 
-size_t prefetch_size( struct prefetch *prefetch){
+size_t prefetch_size(struct prefetch *prefetch)
+{
     if (prefetch) {
 	return prefetch->size;
     } else {
@@ -34,21 +37,25 @@ size_t prefetch_size( struct prefetch *prefetch){
     }
 }
 
-void prefetch_set_is_empty( struct prefetch *prefetch ){
+void prefetch_set_is_empty(struct prefetch *prefetch)
+{
     prefetch_set_full(prefetch, 0);
 }
 
-void prefetch_set_is_full( struct prefetch *prefetch ){
+void prefetch_set_is_full(struct prefetch *prefetch)
+{
     prefetch_set_full(prefetch, 1);
 }
 
-void prefetch_set_full( struct prefetch *prefetch, int val ){
+void prefetch_set_full(struct prefetch *prefetch, int val)
+{
     if (prefetch) {
 	prefetch->is_full = val;
     }
 }
 
-int prefetch_is_full( struct prefetch *prefetch ){
+int prefetch_is_full(struct prefetch *prefetch)
+{
     if (prefetch) {
 	return prefetch->is_full;
     } else {
@@ -56,13 +63,16 @@ int prefetch_is_full( struct prefetch *prefetch ){
     }
 }
 
-int prefetch_contains( struct prefetch *prefetch, uint64_t from, uint32_t len ){
+int prefetch_contains(struct prefetch *prefetch, uint64_t from,
+		      uint32_t len)
+{
     NULLCHECK(prefetch);
     return from >= prefetch->from &&
 	from + len <= prefetch->from + prefetch->len;
 }
 
-char *prefetch_offset( struct prefetch *prefetch, uint64_t from ){
+char *prefetch_offset(struct prefetch *prefetch, uint64_t from)
+{
     NULLCHECK(prefetch);
     return prefetch->buffer + (from - prefetch->from);
 }

src/proxy/prefetch.h

@@ -27,7 +27,8 @@ void prefetch_set_is_empty( struct prefetch *prefetch );
 void prefetch_set_is_full(struct prefetch *prefetch);
 void prefetch_set_full(struct prefetch *prefetch, int val);
 int prefetch_is_full(struct prefetch *prefetch);
-int prefetch_contains( struct prefetch *prefetch, uint64_t from, uint32_t len );
+int prefetch_contains(struct prefetch *prefetch, uint64_t from,
+		      uint32_t len);
 char *prefetch_offset(struct prefetch *prefetch, uint64_t from);
 
 #endif

src/proxy/proxy.c

@@ -13,13 +13,11 @@
 #include <sys/socket.h>
 #include <netinet/tcp.h>
 
-struct proxier* proxy_create(
+struct proxier *proxy_create(char *s_downstream_address,
-	char* s_downstream_address,
 			     char *s_downstream_port,
 			     char *s_upstream_address,
 			     char *s_upstream_port,
-	char* s_upstream_bind,
+			     char *s_upstream_bind, char *s_cache_bytes)
-	char* s_cache_bytes )
 {
     struct proxier *out;
     out = xmalloc(sizeof(struct proxier));
@@ -27,10 +25,9 @@ struct proxier* proxy_create(
     FATAL_IF_NULL(s_downstream_address, "Listen address not specified");
     NULLCHECK(s_downstream_address);
 
-	FATAL_UNLESS(
+    FATAL_UNLESS(parse_to_sockaddr
-		parse_to_sockaddr( &out->listen_on.generic, s_downstream_address ),
+		 (&out->listen_on.generic, s_downstream_address),
-		"Couldn't parse downstream address %s"
+		 "Couldn't parse downstream address %s");
-	);
 
     if (out->listen_on.family != AF_UNIX) {
 	FATAL_IF_NULL(s_downstream_port, "Downstream port not specified");
@@ -41,22 +38,19 @@ struct proxier* proxy_create(
     FATAL_IF_NULL(s_upstream_address, "Upstream address not specified");
     NULLCHECK(s_upstream_address);
 
-	FATAL_UNLESS(
+    FATAL_UNLESS(parse_ip_to_sockaddr
-		parse_ip_to_sockaddr( &out->connect_to.generic, s_upstream_address ),
+		 (&out->connect_to.generic, s_upstream_address),
 		 "Couldn't parse upstream address '%s'",
-		s_upstream_address
+		 s_upstream_address);
-	);
 
     FATAL_IF_NULL(s_upstream_port, "Upstream port not specified");
     NULLCHECK(s_upstream_port);
     parse_port(s_upstream_port, &out->connect_to.v4);
 
     if (s_upstream_bind) {
-		FATAL_IF_ZERO(
+	FATAL_IF_ZERO(parse_ip_to_sockaddr
-			parse_ip_to_sockaddr( &out->connect_from.generic, s_upstream_bind ),
+		      (&out->connect_from.generic, s_upstream_bind),
-			"Couldn't parse bind address '%s'",
+		      "Couldn't parse bind address '%s'", s_upstream_bind);
-			s_upstream_bind
-		);
 	out->bind = 1;
     }
 
@@ -64,6 +58,20 @@ struct proxier* proxy_create(
     out->downstream_fd = -1;
     out->upstream_fd = -1;
 
+    int upstream_timeout = UPSTREAM_TIMEOUT;
+
+    char *env_upstream_timeout = getenv("FLEXNBD_UPSTREAM_TIMEOUT");
+    if (NULL != env_upstream_timeout) {
+	int ut = atoi(env_upstream_timeout);
+        warn("Got %i from atoi\n", ut);
+	if (ut > 0) {
+	    upstream_timeout = ut;
+	}
+    }
+
+    out->upstream_timeout = upstream_timeout;
+    out->upstream_timeout_ms = (long unsigned int) upstream_timeout * 1000;
+
     out->prefetch = NULL;
     if (s_cache_bytes) {
 	int cache_bytes = atoi(s_cache_bytes);
@@ -83,18 +91,21 @@ struct proxier* proxy_create(
     out->req.buf = xmalloc(NBD_MAX_SIZE + NBD_REQUEST_SIZE);
     out->rsp.buf = xmalloc(NBD_MAX_SIZE + NBD_REPLY_SIZE);
 
-	log_context = xmalloc( strlen(s_upstream_address) + strlen(s_upstream_port) + 2 );
+    log_context =
+	xmalloc(strlen(s_upstream_address) + strlen(s_upstream_port) + 2);
     sprintf(log_context, "%s:%s", s_upstream_address, s_upstream_port);
 
     return out;
 }
 
-int proxy_prefetches( struct proxier* proxy ) {
+int proxy_prefetches(struct proxier *proxy)
+{
     NULLCHECK(proxy);
     return proxy->prefetch != NULL;
 }
 
-int proxy_prefetch_bufsize( struct proxier* proxy ){
+int proxy_prefetch_bufsize(struct proxier *proxy)
+{
     NULLCHECK(proxy);
     return prefetch_size(proxy->prefetch);
 }
@@ -110,7 +121,8 @@ void proxy_destroy( struct proxier* proxy )
 }
 
 /* Shared between our two different connect_to_upstream paths */
-void proxy_finish_connect_to_upstream( struct proxier *proxy, uint64_t size );
+void proxy_finish_connect_to_upstream(struct proxier *proxy, uint64_t size,
+				      uint32_t flags);
 
 /* Try to establish a connection to our upstream server. Return 1 on success,
  * 0 on failure. this is a blocking call that returns a non-blocking socket.
@@ -124,22 +136,22 @@ int proxy_connect_to_upstream( struct proxier* proxy )
 
     int fd = socket_connect(&proxy->connect_to.generic, connect_from);
     uint64_t size = 0;
+    uint32_t flags = 0;
 
     if (-1 == fd) {
 	return 0;
     }
 
-	if( !socket_nbd_read_hello( fd, &size ) ) {
+    if (!socket_nbd_read_hello(fd, &size, &flags)) {
-		WARN_IF_NEGATIVE(
+	WARN_IF_NEGATIVE(sock_try_close(fd),
-			sock_try_close( fd ),
+			 "Couldn't close() after failed read of NBD hello on fd %i",
-			"Couldn't close() after failed read of NBD hello on fd %i", fd
+			 fd);
-		);
 	return 0;
     }
 
     proxy->upstream_fd = fd;
     sock_set_nonblock(fd, 1);
-	proxy_finish_connect_to_upstream( proxy, size );
+    proxy_finish_connect_to_upstream(proxy, size, flags);
 
     return 1;
 }
@@ -160,7 +172,8 @@ void proxy_start_connect_to_upstream( struct proxier* proxy )
     fd = socket(to->sa_family, SOCK_STREAM, 0);
 
     if (fd < 0) {
-		warn( SHOW_ERRNO( "Couldn't create socket to reconnect to upstream" ) );
+	warn(SHOW_ERRNO
+	     ("Couldn't create socket to reconnect to upstream"));
 	return;
     }
 
@@ -195,19 +208,28 @@ error:
     return;
 }
 
-void proxy_finish_connect_to_upstream( struct proxier *proxy, uint64_t size ) {
+void proxy_finish_connect_to_upstream(struct proxier *proxy, uint64_t size,
+				      uint32_t flags)
+{
 
     if (proxy->upstream_size == 0) {
 	info("Size of upstream image is %" PRIu64 " bytes", size);
     } else if (proxy->upstream_size != size) {
-		warn(
+	warn("Size changed from %" PRIu64 " to %" PRIu64 " bytes",
-			"Size changed from %"PRIu64" to %"PRIu64" bytes",
+	     proxy->upstream_size, size);
-			proxy->upstream_size, size
-		);
     }
 
     proxy->upstream_size = size;
 
+    if (proxy->upstream_flags == 0) {
+	info("Upstream transmission flags set to %" PRIu32 "", flags);
+    } else if (proxy->upstream_flags != flags) {
+	warn("Upstream transmission flags changed from %" PRIu32 " to %"
+	     PRIu32 "", proxy->upstream_flags, flags);
+    }
+
+    proxy->upstream_flags = flags;
+
     if (AF_UNIX != proxy->connect_to.family) {
 	if (sock_set_tcp_nodelay(proxy->upstream_fd, 1) == -1) {
 	    warn(SHOW_ERRNO("Failed to set TCP_NODELAY"));
@@ -225,11 +247,9 @@ void proxy_disconnect_from_upstream( struct proxier* proxy )
 	info("Closing upstream connection on fd %i", proxy->upstream_fd);
 
 	/* TODO: An NBD disconnect would be pleasant here */
-		WARN_IF_NEGATIVE(
+	WARN_IF_NEGATIVE(sock_try_close(proxy->upstream_fd),
-			sock_try_close( proxy->upstream_fd ),
 			 "Failed to close() fd %i when disconnecting from upstream",
-			proxy->upstream_fd
+			 proxy->upstream_fd);
-		);
 	proxy->upstream_fd = -1;
     }
 }
@@ -241,31 +261,28 @@ void proxy_open_listen_socket(struct proxier* params)
     NULLCHECK(params);
 
     params->listen_fd = socket(params->listen_on.family, SOCK_STREAM, 0);
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(params->listen_fd,
-		params->listen_fd, SHOW_ERRNO( "Couldn't create listen socket" )
+		      SHOW_ERRNO("Couldn't create listen socket")
 	);
 
     /* Allow us to restart quickly */
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(sock_set_reuseaddr(params->listen_fd, 1),
-		sock_set_reuseaddr(params->listen_fd, 1),
 		      SHOW_ERRNO("Couldn't set SO_REUSEADDR")
 	);
 
     if (AF_UNIX != params->listen_on.family) {
-		FATAL_IF_NEGATIVE(
+	FATAL_IF_NEGATIVE(sock_set_tcp_nodelay(params->listen_fd, 1),
-			sock_set_tcp_nodelay(params->listen_fd, 1),
 			  SHOW_ERRNO("Couldn't set TCP_NODELAY")
 	    );
     }
 
-	FATAL_UNLESS_ZERO(
+    FATAL_UNLESS_ZERO(sock_try_bind
-		sock_try_bind( params->listen_fd, &params->listen_on.generic ),
+		      (params->listen_fd, &params->listen_on.generic),
 		      SHOW_ERRNO("Failed to bind to listening socket")
 	);
 
     /* We're only serving one client at a time, hence backlog of 1 */
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(listen(params->listen_fd, 1),
-		listen(params->listen_fd, 1),
 		      SHOW_ERRNO("Failed to listen on listening socket")
 	);
 
@@ -296,8 +313,8 @@ static char* proxy_session_state_names[] = {
 
 static inline int proxy_state_upstream(int state)
 {
-	return state == CONNECT_TO_UPSTREAM || state == READ_INIT_FROM_UPSTREAM ||
+    return state == CONNECT_TO_UPSTREAM || state == READ_INIT_FROM_UPSTREAM
-	       state == WRITE_TO_UPSTREAM   || state == READ_FROM_UPSTREAM;
+	|| state == WRITE_TO_UPSTREAM || state == READ_FROM_UPSTREAM;
 }
 
 int proxy_prefetch_for_request(struct proxier *proxy, int state)
@@ -306,10 +323,12 @@ int proxy_prefetch_for_request( struct proxier* proxy, int state )
     struct nbd_request *req = &proxy->req_hdr;
     struct nbd_reply *rsp = &proxy->rsp_hdr;
 
-	struct nbd_request_raw* req_raw = (struct nbd_request_raw*) proxy->req.buf;
+    struct nbd_request_raw *req_raw =
-	struct nbd_reply_raw *rsp_raw = (struct nbd_reply_raw*) proxy->rsp.buf;
+	(struct nbd_request_raw *) proxy->req.buf;
+    struct nbd_reply_raw *rsp_raw =
+	(struct nbd_reply_raw *) proxy->rsp.buf;
 
-	int is_read = ( req->type & REQUEST_MASK ) == REQUEST_READ;
+    int is_read = req->type == REQUEST_READ;
 
     if (is_read) {
 	/* See if we can respond with what's in our prefetch
@@ -328,11 +347,8 @@ int proxy_prefetch_for_request( struct proxier* proxy, int state )
 	    nbd_h2r_reply(rsp, rsp_raw);
 
 	    /* and the data */
-			memcpy(
+	    memcpy(proxy->rsp.buf + NBD_REPLY_SIZE,
-				proxy->rsp.buf + NBD_REPLY_SIZE,
+		   prefetch_offset(proxy->prefetch, req->from), req->len);
-				prefetch_offset( proxy->prefetch, req->from ),
-				req->len
-			);
 
 	    proxy->rsp.size = NBD_REPLY_SIZE + req->len;
 	    proxy->rsp.needle = 0;
@@ -340,14 +356,14 @@ int proxy_prefetch_for_request( struct proxier* proxy, int state )
 	    /* return early, our work here is done */
 	    return WRITE_TO_DOWNSTREAM;
 	}
-	}
+    } else {
-	else {
 	/* Safety catch.  If we're sending a write request, we
 	 * blow away the cache.  This is very pessimistic, but
 	 * it's simpler (and therefore safer) than working out
 	 * whether we can keep it or not.
 	 */
-		debug( "Blowing away prefetch cache on type %d request.", req->type );
+	debug("Blowing away prefetch cache on type %d request.",
+	      req->type);
 	prefetch_set_is_empty(proxy->prefetch);
     }
 
@@ -395,13 +411,12 @@ int proxy_prefetch_for_reply( struct proxier* proxy, int state )
     prefetched_bytes = proxy->req_hdr.len - proxy->prefetch_req_orig_len;
 
     debug("Prefetched additional %d bytes", prefetched_bytes);
-	memcpy(
+    memcpy(proxy->prefetch->buffer,
-		proxy->prefetch->buffer,
 	   proxy->rsp.buf + proxy->prefetch_req_orig_len + NBD_REPLY_SIZE,
-		prefetched_bytes
+	   prefetched_bytes);
-	);
 
-	proxy->prefetch->from = proxy->req_hdr.from + proxy->prefetch_req_orig_len;
+    proxy->prefetch->from =
+	proxy->req_hdr.from + proxy->prefetch_req_orig_len;
     proxy->prefetch->len = prefetched_bytes;
 
     /* We've finished with proxy->req by now, so don't need to alter it to make
@@ -424,12 +439,14 @@ int proxy_read_from_downstream( struct proxier *proxy, int state )
 {
     ssize_t count;
 
-	struct nbd_request_raw* request_raw = (struct nbd_request_raw*) proxy->req.buf;
+    struct nbd_request_raw *request_raw =
+	(struct nbd_request_raw *) proxy->req.buf;
     struct nbd_request *request = &(proxy->req_hdr);
 
 //      assert( state == READ_FROM_DOWNSTREAM );
 
-	count = iobuf_read( proxy->downstream_fd, &proxy->req, NBD_REQUEST_SIZE );
+    count =
+	iobuf_read(proxy->downstream_fd, &proxy->req, NBD_REQUEST_SIZE);
 
     if (count == -1) {
 	warn(SHOW_ERRNO("Couldn't read request from downstream"));
@@ -439,7 +456,7 @@ int proxy_read_from_downstream( struct proxier *proxy, int state )
     if (proxy->req.needle == NBD_REQUEST_SIZE) {
 	nbd_r2h_request(request_raw, request);
 
-		if ( ( request->type & REQUEST_MASK ) == REQUEST_DISCONNECT ) {
+	if (request->type == REQUEST_DISCONNECT) {
 	    info("Received disconnect request from client");
 	    return EXIT;
 	}
@@ -448,15 +465,17 @@ int proxy_read_from_downstream( struct proxier *proxy, int state )
 	 * been taken into account in the xmalloc, so no need to worry
 	 * about them here
 	 */
-		if ( ( request->type & REQUEST_MASK ) == REQUEST_READ ) {
+	if (request->type == REQUEST_READ) {
 	    if (request->len > NBD_MAX_SIZE) {
-				warn( "NBD read request size %"PRIu32" too large", request->len );
+		warn("NBD read request size %" PRIu32 " too large",
+		     request->len);
 		return EXIT;
 	    }
 	}
-		if ( (request->type & REQUEST_MASK ) == REQUEST_WRITE ) {
+	if (request->type == REQUEST_WRITE) {
 	    if (request->len > NBD_MAX_SIZE) {
-				warn( "NBD write request size %"PRIu32" too large", request->len );
+		warn("NBD write request size %" PRIu32 " too large",
+		     request->len);
 		return EXIT;
 	    }
 
@@ -465,10 +484,9 @@ int proxy_read_from_downstream( struct proxier *proxy, int state )
     }
 
     if (proxy->req.needle == proxy->req.size) {
-		debug(
+	debug("Received NBD request from downstream. type=%" PRIu16
-			"Received NBD request from downstream. type=%"PRIu32" from=%"PRIu64" len=%"PRIu32,
+	      " flags=%" PRIu16 " from=%" PRIu64 " len=%" PRIu32,
-			request->type, request->from, request->len
+	      request->type, request->flags, request->from, request->len);
-		);
 
 	/* Finished reading, so advance state. Leave size untouched so the next
 	 * state knows how many bytes to write */
@@ -486,9 +504,8 @@ int proxy_continue_connecting_to_upstream( struct proxier* proxy, int state )
 
 //      assert( state == CONNECT_TO_UPSTREAM );
 
-	result = getsockopt(
+    result =
-		proxy->upstream_fd, SOL_SOCKET, SO_ERROR,  &error, &len
+	getsockopt(proxy->upstream_fd, SOL_SOCKET, SO_ERROR, &error, &len);
-	);
 
     if (result == -1) {
 	warn(SHOW_ERRNO("Failed to tell if connected to upstream"));
@@ -504,6 +521,20 @@ int proxy_continue_connecting_to_upstream( struct proxier* proxy, int state )
     /* Data may have changed while we were disconnected */
     prefetch_set_is_empty(proxy->prefetch);
 
+    /* Reset our needles and sizes.
+     *
+     * Don't zero the req buffer size in case there's an outstanding request
+     * waiting to be re-sent following a disconnection.  The init and rsp
+     * buffers are used for reading from upstream.  If we're in this state then
+     * any upstream reads will be re-requested.
+     */
+    proxy->init.needle = 0;
+    proxy->init.size = 0;
+    proxy->rsp.needle = 0;
+    proxy->rsp.size = 0;
+    proxy->req.needle = 0;
+    /* Don't zero the req.size, as we may need to re-write it upstream */
+
     info("Connected to upstream on fd %i", proxy->upstream_fd);
     return READ_INIT_FROM_UPSTREAM;
 }
@@ -514,33 +545,40 @@ int proxy_read_init_from_upstream( struct proxier* proxy, int state )
 
 //      assert( state == READ_INIT_FROM_UPSTREAM );
 
-	count = iobuf_read( proxy->upstream_fd, &proxy->init, sizeof( struct nbd_init_raw ) );
+    count =
+	iobuf_read(proxy->upstream_fd, &proxy->init,
+		   sizeof(struct nbd_init_raw));
 
     if (count == -1) {
 	warn(SHOW_ERRNO("Failed to read init from upstream"));
-		goto disconnect;
+	return CONNECT_TO_UPSTREAM;
     }
 
     if (proxy->init.needle == proxy->init.size) {
 	uint64_t upstream_size;
-		if ( !nbd_check_hello( (struct nbd_init_raw*) proxy->init.buf, &upstream_size ) ) {
+	uint32_t upstream_flags;
+	if (!nbd_check_hello
+	    ((struct nbd_init_raw *) proxy->init.buf, &upstream_size,
+	     &upstream_flags)) {
 	    warn("Upstream sent invalid init");
-			goto disconnect;
+	    return CONNECT_TO_UPSTREAM;
 	}
 
-		/* Currently, we only get disconnected from upstream (so needing to come
+	/* record the flags, and log the reconnection, set TCP_NODELAY */
-		 * here) when we have an outstanding request. If that becomes false,
+	proxy_finish_connect_to_upstream(proxy, upstream_size,
-		 * we'll need to choose the right state to return to here */
+					 upstream_flags);
+
+	/* Finished reading the init response now, reset the needle. */
 	proxy->init.needle = 0;
+
+	/* Currently, we only get disconnected from upstream (so needing to
+	 * come here) when we have an outstanding request. If that becomes
+	 * false, we'll need to choose the right state to return to here.
+	 */
 	return WRITE_TO_UPSTREAM;
     }
 
     return state;
-
-disconnect:
-	proxy->init.needle = 0;
-	proxy->init.size = 0;
-	return CONNECT_TO_UPSTREAM;
 }
 
 int proxy_write_to_upstream(struct proxier *proxy, int state)
@@ -562,7 +600,6 @@ int proxy_write_to_upstream( struct proxier* proxy, int state )
 
     if (count == -1) {
 	warn(SHOW_ERRNO("Failed to send request to upstream"));
-		proxy->req.needle = 0;
 	// We're throwing the socket away so no need to uncork
 	return CONNECT_TO_UPSTREAM;
     }
@@ -591,14 +628,15 @@ int proxy_read_from_upstream( struct proxier* proxy, int state )
     ssize_t count;
 
     struct nbd_reply *reply = &(proxy->rsp_hdr);
-	struct nbd_reply_raw* reply_raw = (struct nbd_reply_raw*) proxy->rsp.buf;
+    struct nbd_reply_raw *reply_raw =
+	(struct nbd_reply_raw *) proxy->rsp.buf;
 
     /* We can't assume the NBD_REPLY_SIZE + req->len is what we'll get back */
     count = iobuf_read(proxy->upstream_fd, &proxy->rsp, NBD_REPLY_SIZE);
 
     if (count == -1) {
 	warn(SHOW_ERRNO("Failed to get reply from upstream"));
-		goto disconnect;
+	return CONNECT_TO_UPSTREAM;
     }
 
     if (proxy->rsp.needle == NBD_REPLY_SIZE) {
@@ -606,15 +644,10 @@ int proxy_read_from_upstream( struct proxier* proxy, int state )
 
 	if (reply->magic != REPLY_MAGIC) {
 	    warn("Reply magic is incorrect");
-			goto disconnect;
+	    return CONNECT_TO_UPSTREAM;
 	}
 
-		if ( reply->error != 0 ) {
+	if (proxy->req_hdr.type == REQUEST_READ) {
-			warn( "NBD error returned from upstream: %"PRIu32, reply->error );
-			goto disconnect;
-		}
-
-		if ( ( proxy->req_hdr.type & REQUEST_MASK ) == REQUEST_READ ) {
 	    /* Get the read reply data too. */
 	    proxy->rsp.size += proxy->req_hdr.len;
 	}
@@ -627,11 +660,6 @@ int proxy_read_from_upstream( struct proxier* proxy, int state )
     }
 
     return state;
-
-disconnect:
-	proxy->rsp.needle = 0;
-	proxy->rsp.size = 0;
-	return CONNECT_TO_UPSTREAM;
 }
 
 
@@ -690,7 +718,8 @@ void proxy_session( struct proxier* proxy )
 
 
     /* First action: Write hello to downstream */
-	nbd_hello_to_buf( (struct nbd_init_raw *) proxy->rsp.buf, proxy->upstream_size );
+    nbd_hello_to_buf((struct nbd_init_raw *) proxy->rsp.buf,
+		     proxy->upstream_size, proxy->upstream_flags);
     proxy->rsp.size = sizeof(struct nbd_init_raw);
     proxy->rsp.needle = 0;
     state = WRITE_TO_DOWNSTREAM;
@@ -718,13 +747,13 @@ void proxy_session( struct proxier* proxy )
 	if (state != old_state) {
 	    state_started = monotonic_time_ms();
 
-			debug(
+	    debug("State transition from %s to %s",
-				"State transition from %s to %s",
 		  proxy_session_state_names[old_state],
 		  proxy_session_state_names[state]
 		);
 	} else {
-			debug( "Proxy is in state %s", proxy_session_state_names[state], state );
+	    debug("Proxy is in state %s", proxy_session_state_names[state],
+		  state);
 	}
 
 	old_state = state;
@@ -766,10 +795,15 @@ void proxy_session( struct proxier* proxy )
 	};
 
 	if (select_timeout.tv_sec > 0) {
+	    if (select_timeout.tv_sec > proxy->upstream_timeout) {
+		select_timeout.tv_sec = proxy->upstream_timeout;
+	    }
 	    select_timeout_ptr = &select_timeout;
 	}
 
-		result = sock_try_select( FD_SETSIZE, &rfds, &wfds, NULL, select_timeout_ptr );
+	result =
+	    sock_try_select(FD_SETSIZE, &rfds, &wfds, NULL,
+			    select_timeout_ptr);
 
 	if (result == -1) {
 	    warn(SHOW_ERRNO("select() failed: "));
@@ -795,7 +829,8 @@ void proxy_session( struct proxier* proxy )
 	    break;
 	case CONNECT_TO_UPSTREAM:
 	    if (FD_ISSET(proxy->upstream_fd, &wfds)) {
-					state = proxy_continue_connecting_to_upstream( proxy, state );
+		state =
+		    proxy_continue_connecting_to_upstream(proxy, state);
 	    }
 	    /* Leaving state untouched will retry connecting to upstream -
 	     * so introduce a bit of sleep  */
@@ -836,27 +871,15 @@ void proxy_session( struct proxier* proxy )
 	/* In these states, we're interested in restarting after a timeout.
 	 */
 	if (old_state == state && proxy_state_upstream(state)) {
-			if ( ( monotonic_time_ms() ) - state_started > UPSTREAM_TIMEOUT ) {
+	    if ((monotonic_time_ms()) - state_started > proxy->upstream_timeout_ms) {
-				warn(
+		warn("Timed out in state %s while communicating with upstream", proxy_session_state_names[state]);
-					"Timed out in state %s while communicating with upstream",
-					proxy_session_state_names[state]
-				);
 		state = CONNECT_TO_UPSTREAM;
-
-				/* Since we've timed out, we won't have gone through the timeout logic
-				 * in the various state handlers that resets these appropriately... */
-				proxy->init.size = 0;
-				proxy->init.needle = 0;
-				proxy->rsp.size = 0;
-				proxy->rsp.needle = 0;
 	    }
 	}
     }
 
-	info(
+    info("Finished proxy session on fd %i after %" PRIu64
-		"Finished proxy session on fd %i after %"PRIu64" successful request(s)",
+	 " successful request(s)", proxy->downstream_fd, proxy->req_count);
-		proxy->downstream_fd, proxy->req_count
-	);
 
     /* Reset these two for the next session */
     proxy->req_count = 0;
@@ -881,13 +904,13 @@ int proxy_accept( struct proxier* params )
     FD_ZERO(&fds);
     FD_SET(params->listen_fd, &fds);
 
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(sock_try_select(FD_SETSIZE, &fds, NULL, NULL, NULL),
-		sock_try_select(FD_SETSIZE, &fds, NULL, NULL, NULL),
 		      SHOW_ERRNO("select() failed")
 	);
 
     if (FD_ISSET(params->listen_fd, &fds)) {
-		client_fd = accept( params->listen_fd, &client_address.generic, &socklen );
+	client_fd =
+	    accept(params->listen_fd, &client_address.generic, &socklen);
 
 	if (client_address.family != AF_UNIX) {
 	    if (sock_set_tcp_nodelay(client_fd, 1) == -1) {
@@ -900,11 +923,9 @@ int proxy_accept( struct proxier* params )
 	params->downstream_fd = client_fd;
 	proxy_session(params);
 
-		WARN_IF_NEGATIVE(
+	WARN_IF_NEGATIVE(sock_try_close(params->downstream_fd),
-			sock_try_close( params->downstream_fd ),
 			 "Couldn't close() downstram fd %i after proxy session",
-			params->downstream_fd
+			 params->downstream_fd);
-		);
 	params->downstream_fd = -1;
     }
 
@@ -929,33 +950,31 @@ void proxy_cleanup( struct proxier* proxy )
 
 	if (AF_UNIX == proxy->listen_on.family) {
 	    if (-1 == unlink(proxy->listen_on.un.sun_path)) {
-				warn( SHOW_ERRNO( "Failed to unlink %s", proxy->listen_on.un.sun_path ) );
+		warn(SHOW_ERRNO
+		     ("Failed to unlink %s",
+		      proxy->listen_on.un.sun_path));
 	    }
 	}
 
-		WARN_IF_NEGATIVE(
+	WARN_IF_NEGATIVE(sock_try_close(proxy->listen_fd),
-			sock_try_close( proxy->listen_fd ),
+			 SHOW_ERRNO("Failed to close() listen fd %i",
-			SHOW_ERRNO( "Failed to close() listen fd %i", proxy->listen_fd )
+				    proxy->listen_fd)
 	    );
 	proxy->listen_fd = -1;
     }
 
     if (-1 != proxy->downstream_fd) {
-		WARN_IF_NEGATIVE(
+	WARN_IF_NEGATIVE(sock_try_close(proxy->downstream_fd),
-			sock_try_close( proxy->downstream_fd ),
+			 SHOW_ERRNO("Failed to close() downstream fd %i",
-			SHOW_ERRNO(
+				    proxy->downstream_fd)
-				"Failed to close() downstream fd %i", proxy->downstream_fd
-			)
 	    );
 	proxy->downstream_fd = -1;
     }
 
     if (-1 != proxy->upstream_fd) {
-		WARN_IF_NEGATIVE(
+	WARN_IF_NEGATIVE(sock_try_close(proxy->upstream_fd),
-			sock_try_close( proxy->upstream_fd ),
+			 SHOW_ERRNO("Failed to close() upstream fd %i",
-			SHOW_ERRNO(
+				    proxy->upstream_fd)
-				"Failed to close() upstream fd %i", proxy->upstream_fd
-			)
 	    );
 	proxy->upstream_fd = -1;
     }
@@ -982,4 +1001,3 @@ int do_proxy( struct proxier* params )
 
     return 0;
 }
-

src/proxy/proxy.h

@@ -14,10 +14,10 @@
 #endif
 
 /** UPSTREAM_TIMEOUT
- * How long ( in ms ) to allow for upstream to respond. If it takes longer
+ * How long (in s) to allow for upstream to respond. If it takes longer
  * than this, we will cancel the current request-response to them and resubmit
  */
-#define UPSTREAM_TIMEOUT 30 * 1000
+#define UPSTREAM_TIMEOUT 30
 
 struct proxier {
 	/** address/port to bind to */
@@ -46,6 +46,9 @@ struct proxier {
     /* This is the size we advertise to the downstream server */
     uint64_t upstream_size;
 
+    /* These are the transmission flags sent as part of the handshake */
+    uint32_t upstream_flags;
+
     /* We transform the raw request header into here */
     struct nbd_request req_hdr;
 
@@ -69,6 +72,16 @@ struct proxier {
     uint64_t req_count;
     int hello_sent;
 
+    /*
+     * How long (in s) to allow for upstream to respond. If it takes longer
+     * than this, we will cancel the current request-response to them and
+     * resubmit
+     *
+     * Defaults to UPSTREAM_TIMEOUT but can be overridden in the environment.
+     */
+     int upstream_timeout;
+     unsigned long int upstream_timeout_ms;
+
 	/** These are only used if we pass --cache on the command line */
 
     /* While the in-flight request has been munged by prefetch, these two are
@@ -82,16 +95,13 @@ struct proxier {
 	/** */
 };
 
-struct proxier* proxy_create(
+struct proxier *proxy_create(char *s_downstream_address,
-	char* s_downstream_address,
 			     char *s_downstream_port,
 			     char *s_upstream_address,
 			     char *s_upstream_port,
-	char* s_upstream_bind,
+			     char *s_upstream_bind, char *s_cache_bytes);
-	char* s_cache_bytes);
 int do_proxy(struct proxier *proxy);
 void proxy_cleanup(struct proxier *proxy);
 void proxy_destroy(struct proxier *proxy);
 
 #endif
-

src/server/acl.c

@@ -22,7 +22,9 @@ static int testmasks[9] = { 0,128,192,224,240,248,252,254,255 };
 /** Test whether AF_INET or AF_INET6 sockaddr is included in the given access
   * control list, returning 1 if it is, and 0 if not.
   */
-static int is_included_in_acl(int list_length, struct ip_and_mask (*list)[], union mysockaddr* test)
+static int is_included_in_acl(int list_length,
+			      struct ip_and_mask (*list)[],
+			      union mysockaddr *test)
 {
     NULLCHECK(test);
 
@@ -33,7 +35,8 @@ static int is_included_in_acl(int list_length, struct ip_and_mask (*list)[], uni
 	int testbits;
 	unsigned char *raw_address1 = NULL, *raw_address2 = NULL;
 
-		debug("checking acl entry %d (%d/%d)", i, test->generic.sa_family, entry->ip.family);
+	debug("checking acl entry %d (%d/%d)", i, test->generic.sa_family,
+	      entry->ip.family);
 
 	if (test->generic.sa_family != entry->ip.family) {
 	    continue;
@@ -43,24 +46,24 @@ static int is_included_in_acl(int list_length, struct ip_and_mask (*list)[], uni
 	    debug("it's an AF_INET");
 	    raw_address1 = (unsigned char *) &test->v4.sin_addr;
 	    raw_address2 = (unsigned char *) &entry->ip.v4.sin_addr;
-		}
+	} else if (test->generic.sa_family == AF_INET6) {
-		else if (test->generic.sa_family == AF_INET6) {
 	    debug("it's an AF_INET6");
 	    raw_address1 = (unsigned char *) &test->v6.sin6_addr;
 	    raw_address2 = (unsigned char *) &entry->ip.v6.sin6_addr;
-		}
+	} else {
-		else {
 	    fatal("Can't check an ACL for this address type.");
 	}
 
 	debug("testbits=%d", entry->mask);
 
 	for (testbits = entry->mask; testbits > 0; testbits -= 8) {
-			debug("testbits=%d, c1=%02x, c2=%02x", testbits, raw_address1[0], raw_address2[0]);
+	    debug("testbits=%d, c1=%02x, c2=%02x", testbits,
+		  raw_address1[0], raw_address2[0]);
 	    if (testbits >= 8) {
-				if (raw_address1[0] != raw_address2[0]) { goto no_match; }
+		if (raw_address1[0] != raw_address2[0]) {
+		    goto no_match;
 		}
-			else {
+	    } else {
 		if ((raw_address1[0] & testmasks[testbits % 8]) !=
 		    (raw_address2[0] & testmasks[testbits % 8])) {
 		    goto no_match;
@@ -86,8 +89,7 @@ int acl_includes( struct acl * acl, union mysockaddr * addr )
 
     if (0 == acl->len) {
 	return !(acl->default_deny);
-	}
+    } else {
-	else {
 	return is_included_in_acl(acl->len, acl->entries, addr);
     }
 }
@@ -105,4 +107,3 @@ void acl_destroy( struct acl * acl )
     acl->entries = NULL;
     free(acl);
 }
-

src/server/bitset.h

@@ -30,30 +30,41 @@ typedef bitfield_word_t * bitfield_p;
 			((_bytes + (BITFIELD_WORD_SIZE-1)) / BITFIELD_WORD_SIZE)
 
 /** Return the bit value ''idx'' in array ''b'' */
-static inline int bit_get(bitfield_p b, uint64_t idx) {
+static inline int bit_get(bitfield_p b, uint64_t idx)
+{
     return (BIT_WORD(b, idx) >> (idx & (BITS_PER_WORD - 1))) & 1;
 }
 
 /** Return 1 if the bit at ''idx'' in array ''b'' is set */
-static inline int bit_is_set(bitfield_p b, uint64_t idx) {
+static inline int bit_is_set(bitfield_p b, uint64_t idx)
+{
     return bit_get(b, idx);
 }
+
 /** Return 1 if the bit at ''idx'' in array ''b'' is clear */
-static inline int bit_is_clear(bitfield_p b, uint64_t idx) {
+static inline int bit_is_clear(bitfield_p b, uint64_t idx)
+{
     return !bit_get(b, idx);
 }
+
 /** Tests whether the bit at ''idx'' in array ''b'' has value ''value'' */
-static inline int bit_has_value(bitfield_p b, uint64_t idx, int value) {
+static inline int bit_has_value(bitfield_p b, uint64_t idx, int value)
+{
     return bit_get(b, idx) == ! !value;
 }
+
 /** Sets the bit ''idx'' in array ''b'' */
-static inline void bit_set(bitfield_p b, uint64_t idx) {
+static inline void bit_set(bitfield_p b, uint64_t idx)
+{
     BIT_WORD(b, idx) |= BIT_MASK(idx);
 }
+
 /** Clears the bit ''idx'' in array ''b'' */
-static inline void bit_clear(bitfield_p b, uint64_t idx) {
+static inline void bit_clear(bitfield_p b, uint64_t idx)
+{
     BIT_WORD(b, idx) &= ~BIT_MASK(idx);
 }
+
 /** Sets ''len'' bits in array ''b'' starting at offset ''from'' */
 static inline void bit_set_range(bitfield_p b, uint64_t from, uint64_t len)
 {
@@ -72,8 +83,10 @@ static inline void bit_set_range(bitfield_p b, uint64_t from, uint64_t len)
 	bit_set(b, from++);
     }
 }
+
 /** Clears ''len'' bits in array ''b'' starting at offset ''from'' */
-static inline void bit_clear_range(bitfield_p b, uint64_t from, uint64_t len)
+static inline void bit_clear_range(bitfield_p b, uint64_t from,
+				   uint64_t len)
 {
     for (; (from % BITS_PER_WORD) != 0 && len > 0; len--) {
 	bit_clear(b, from++);
@@ -96,7 +109,9 @@ static inline void bit_clear_range(bitfield_p b, uint64_t from, uint64_t len)
   * bits that are the same as the first one specified. If ''run_is_set'' is
   * non-NULL, the value of that bit is placed into it.
   */
-static inline uint64_t bit_run_count(bitfield_p b, uint64_t from, uint64_t len, int *run_is_set) {
+static inline uint64_t bit_run_count(bitfield_p b, uint64_t from,
+				     uint64_t len, int *run_is_set)
+{
     uint64_t count = 0;
     int first_value = bit_get(b, from);
     bitfield_word_t word_match = first_value ? -1 : 0;
@@ -185,8 +200,10 @@ static inline struct bitset *bitset_alloc( uint64_t size, int resolution )
     // calculate a size to allocate that is a multiple of the size of the
     // bitfield word
     size_t bitfield_size =
-			BIT_WORDS_FOR_SIZE((( size + resolution - 1 ) / resolution)) * sizeof( bitfield_word_t );
+	BIT_WORDS_FOR_SIZE(((size + resolution -
-	struct bitset *bitset = xmalloc(sizeof( struct bitset ) + ( bitfield_size / 8 ) );
+			     1) / resolution)) * sizeof(bitfield_word_t);
+    struct bitset *bitset =
+	xmalloc(sizeof(struct bitset) + (bitfield_size / 8));
 
     bitset->size = size;
     bitset->resolution = resolution;
@@ -224,12 +241,9 @@ static inline void bitset_free( struct bitset * set )
   FATAL_IF_NEGATIVE(pthread_mutex_unlock(&set->lock), "Error unlocking bitset")
 
 
-static inline void bitset_stream_enqueue(
+static inline void bitset_stream_enqueue(struct bitset *set,
-	struct bitset * set,
 					 enum bitset_stream_events event,
-	uint64_t from,
+					 uint64_t from, uint64_t len)
-	uint64_t len
-)
 {
     struct bitset_stream *stream = set->stream;
 
@@ -254,10 +268,8 @@ static inline void bitset_stream_enqueue(
     return;
 }
 
-static inline void bitset_stream_dequeue(
+static inline void bitset_stream_dequeue(struct bitset *set,
-	struct bitset * set,
+					 struct bitset_stream_entry *out)
-	struct bitset_stream_entry * out
-)
 {
     struct bitset_stream *stream = set->stream;
     struct bitset_stream_entry *dequeued;
@@ -298,10 +310,9 @@ static inline size_t bitset_stream_size( struct bitset * set )
     return size;
 }
 
-static inline uint64_t bitset_stream_queued_bytes(
+static inline uint64_t bitset_stream_queued_bytes(struct bitset *set,
-	struct bitset * set,
+						  enum bitset_stream_events
-	enum bitset_stream_events event
+						  event)
-)
 {
     uint64_t total;
 
@@ -331,10 +342,8 @@ static inline void bitset_disable_stream( struct bitset * set  )
 /** Set the bits in a bitset which correspond to the given bytes in the larger
   * file.
   */
-static inline void bitset_set_range(
+static inline void bitset_set_range(struct bitset *set,
-	struct bitset * set,
+				    uint64_t from, uint64_t len)
-	uint64_t from,
-	uint64_t len)
 {
     INT_FIRST_AND_LAST;
     BITSET_LOCK;
@@ -357,10 +366,8 @@ static inline void bitset_set( struct bitset * set )
 /** Clear the bits in a bitset which correspond to the given bytes in the
   * larger file.
   */
-static inline void bitset_clear_range(
+static inline void bitset_clear_range(struct bitset *set,
-	struct bitset * set,
+				      uint64_t from, uint64_t len)
-	uint64_t from,
-	uint64_t len)
 {
     INT_FIRST_AND_LAST;
     BITSET_LOCK;
@@ -383,12 +390,9 @@ static inline void bitset_clear( struct bitset * set )
 /** As per bitset_run_count but also tells you whether the run it found was set
   * or unset, atomically.
   */
-static inline uint64_t bitset_run_count_ex(
+static inline uint64_t bitset_run_count_ex(struct bitset *set,
-	struct bitset * set,
 					   uint64_t from,
-	uint64_t len,
+					   uint64_t len, int *run_is_set)
-	int* run_is_set
-)
 {
     uint64_t run;
 
@@ -401,7 +405,9 @@ static inline uint64_t bitset_run_count_ex(
     INT_FIRST_AND_LAST;
 
     BITSET_LOCK;
-	run = bit_run_count(set->bits, first, bitlen, run_is_set) * set->resolution;
+    run =
+	bit_run_count(set->bits, first, bitlen,
+		      run_is_set) * set->resolution;
     run -= (from % set->resolution);
     BITSET_UNLOCK;
 
@@ -411,10 +417,8 @@ static inline uint64_t bitset_run_count_ex(
 /** Counts the number of contiguous bytes that are represented as a run in
   * the bit field.
   */
-static inline uint64_t bitset_run_count(
+static inline uint64_t bitset_run_count(struct bitset *set,
-	struct bitset * set,
+					uint64_t from, uint64_t len)
-	uint64_t from,
-	uint64_t len)
 {
     return bitset_run_count_ex(set, from, len, NULL);
 }
@@ -435,4 +439,3 @@ static inline int bitset_is_set_at( struct bitset * set, uint64_t at )
 
 
 #endif
-

src/server/client.c

@@ -18,14 +18,16 @@
 
 // When this signal is invoked, we call shutdown() on the client fd, which
 // results in the thread being wound up
-void client_killswitch_hit(int signal __attribute__ ((unused)), siginfo_t *info, void *ptr __attribute__ ((unused)))
+void client_killswitch_hit(int signal
+			   __attribute__ ((unused)), siginfo_t * info,
+			   void *ptr __attribute__ ((unused)))
 {
     int fd = info->si_value.sival_int;
     warn("Killswitch for fd %i activated, calling shutdown on socket", fd);
 
-	FATAL_IF(
+    FATAL_IF(-1 == shutdown(fd, SHUT_RDWR),
-		-1 == shutdown( fd, SHUT_RDWR ),
+	     SHOW_ERRNO
-		SHOW_ERRNO( "Failed to shutdown() the socket, killing the server" )
+	     ("Failed to shutdown() the socket, killing the server")
 	);
 }
 
@@ -53,8 +55,8 @@ struct client *client_create( struct server *serve, int socket )
 
     c->stop_signal = self_pipe_create();
 
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(timer_create
-		timer_create( CLOCK_MONOTONIC, &evp, &(c->killswitch) ),
+		      (CLOCK_MONOTONIC, &evp, &(c->killswitch)),
 		      SHOW_ERRNO("Failed to create killswitch timer")
 	);
 
@@ -67,7 +69,8 @@ void client_signal_stop( struct client *c)
 {
     NULLCHECK(c);
 
-	debug("client %p: signal stop (%d, %d)", c,c->stop_signal->read_fd, c->stop_signal->write_fd );
+    debug("client %p: signal stop (%d, %d)", c, c->stop_signal->read_fd,
+	  c->stop_signal->write_fd);
     self_pipe_signal(c->stop_signal);
 }
 
@@ -75,8 +78,7 @@ void client_destroy( struct client *client )
 {
     NULLCHECK(client);
 
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(timer_delete(client->killswitch),
-		timer_delete( client->killswitch ),
 		      SHOW_ERRNO("Couldn't delete killswitch")
 	);
 
@@ -119,7 +121,8 @@ void write_not_zeroes(struct client* client, uint64_t from, uint64_t len)
 
 	uint64_t run = bitset_run_count(map, from, len);
 
-		debug("write_not_zeroes: from=%ld, len=%d, run=%d", from, len, run);
+	debug("write_not_zeroes: from=%ld, len=%d, run=%d", from, len,
+	      run);
 
 	if (run > len) {
 	    run = len;
@@ -164,8 +167,7 @@ void write_not_zeroes(struct client* client, uint64_t from, uint64_t len)
 	    bitset_set_range(map, from, run);
 	    len -= run;
 	    from += run;
-		}
+	} else {
-		else {
 	    char zerobuffer[block_allocation_resolution];
 	    /* not allocated, read in block_allocation_resoution */
 	    while (run > 0) {
@@ -183,7 +185,8 @@ void write_not_zeroes(struct client* client, uint64_t from, uint64_t len)
 		 */
 
 		int all_zeros = (zerobuffer[0] == 0) &&
-					(0 == memcmp( zerobuffer, zerobuffer+1, blockrun-1 ));
+		    (0 ==
+		     memcmp(zerobuffer, zerobuffer + 1, blockrun - 1));
 
 		if (!all_zeros) {
 		    memcpy(client->mapped + from, zerobuffer, blockrun);
@@ -217,7 +220,8 @@ int fd_read_request( int fd, struct nbd_request_raw *out_request)
 
 /* Returns 1 if *request was filled with a valid request which we should
  * try to honour. 0 otherwise. */
-int client_read_request( struct client * client , struct nbd_request *out_request, int * disconnected )
+int client_read_request(struct client *client,
+			struct nbd_request *out_request, int *disconnected)
 {
     NULLCHECK(client);
     NULLCHECK(out_request);
@@ -228,11 +232,13 @@ int client_read_request( struct client * client , struct nbd_request *out_reques
 	*disconnected = 1;
 	switch (errno) {
 	case 0:
-				debug( "EOF while reading request" );
+	    warn("EOF while reading request");
 	    return 0;
 	case ECONNRESET:
-				debug( "Connection reset while"
+	    warn("Connection reset while" " reading request");
-						" reading request" );
+	    return 0;
+	case ETIMEDOUT:
+	    warn("Connection timed out while" " reading request");
 	    return 0;
 	default:
 	    /* FIXME: I've seen this happen, but I
@@ -242,9 +248,7 @@ int client_read_request( struct client * client , struct nbd_request *out_reques
 	     * again.  It should *probably* be an
 	     * error() call, but I want to be sure.
 	     * */
-				fatal("Error reading request: %d, %s",
+	    fatal("Error reading request: %d, %s", errno, strerror(errno));
-						errno,
-						strerror( errno ));
 	}
     }
 
@@ -289,7 +293,8 @@ int fd_write_reply( int fd, uint64_t handle, int error )
  * Returns 1; we don't check for errors on the write.
  * TODO: Check for errors on the write.
  */
-int client_write_reply( struct client * client, struct nbd_request *request, int error )
+int client_write_reply(struct client *client, struct nbd_request *request,
+		       int error)
 {
     return fd_write_reply(client->socket, request->handle.w, error);
 }
@@ -303,14 +308,17 @@ void client_write_init( struct client * client, uint64_t size )
     memcpy(init.passwd, INIT_PASSWD, sizeof(init.passwd));
     init.magic = INIT_MAGIC;
     init.size = size;
-	memset( init.reserved, 0, 128 );
+    /* As more features are implemented, this is the place to advertise
+     * them.
+     */
+    init.flags = FLAG_HAS_FLAGS | FLAG_SEND_FLUSH | FLAG_SEND_FUA;
+    memset(init.reserved, 0, 124);
 
     nbd_h2r_init(&init, &init_raw);
 
-	ERROR_IF_NEGATIVE(
+    ERROR_IF_NEGATIVE(writeloop
-		writeloop(client->socket, &init_raw, sizeof(init_raw)),
+		      (client->socket, &init_raw, sizeof(init_raw)),
-		"Couldn't send hello"
+		      "Couldn't send hello");
-	);
 }
 
 
@@ -330,23 +338,15 @@ void client_flush( struct client * client, size_t len )
     size_t spliced = 0;
 
     while (spliced < len) {
-		ssize_t received = splice(
+	ssize_t received = splice(client->socket, NULL,
-				client->socket, NULL,
 				  pipes[1], NULL,
 				  len - spliced, flags);
-		FATAL_IF_NEGATIVE( received,
+	FATAL_IF_NEGATIVE(received, "splice error: %s", strerror(errno));
-				"splice error: %s",
-				strerror(errno));
 	ssize_t junked = 0;
 	while (junked < received) {
 	    ssize_t junk;
-			junk = splice(
+	    junk = splice(pipes[0], NULL, devnull, NULL, received, flags);
-				pipes[0], NULL,
+	    FATAL_IF_NEGATIVE(junk, "splice error: %s", strerror(errno));
-				devnull, NULL,
-				received, flags );
-			FATAL_IF_NEGATIVE( junk,
-				"splice error: %s",
-				strerror(errno));
 	    junked += junk;
 	}
 	spliced += received;
@@ -378,27 +378,26 @@ int client_request_needs_reply( struct client * client,
 	return 0;
     }
 
-	debug(
+    debug("request type=%" PRIu16 ", flags=%" PRIu16 ", from=%" PRIu64
-		"request type=%"PRIu32", from=%"PRIu64", len=%"PRIu32", handle=0x%08X",
+	  ", len=%" PRIu32 ", handle=0x%08X", request.type, request.flags,
-		request.type, request.from, request.len, request.handle
+	  request.from, request.len, request.handle);
-	);
 
-	/* check it's not out of range */
+    /* check it's not out of range. NBD protocol requires ENOSPC to be
+     * returned in this instance 
+     */
     if (request.from + request.len > client->serve->size) {
 	warn("write request %" PRIu64 "+%" PRIu32 " out of range",
-		  request.from, request.len
+	     request.from, request.len);
-		);
 	if (request.type == REQUEST_WRITE) {
 	    client_flush(client, request.len);
 	}
-		client_write_reply( client, &request, EPERM ); /* TODO: Change to ERANGE ? */
+	client_write_reply(client, &request, ENOSPC);
 	client->disconnect = 0;
 	return 0;
     }
 
 
-	switch (request.type)
+    switch (request.type) {
-	{
     case REQUEST_READ:
 	break;
     case REQUEST_WRITE:
@@ -407,15 +406,22 @@ int client_request_needs_reply( struct client * client,
 	debug("request disconnect");
 	client->disconnect = 1;
 	return 0;
-
+    case REQUEST_FLUSH:
+	break;
     default:
-		fatal("Unknown request 0x%08X", request.type);
+	/* NBD prototcol says servers SHOULD return EINVAL to unknown
+	 * commands */
+	warn("Unknown request 0x%08X", request.type);
+	client_write_reply(client, &request, EINVAL);
+	client->disconnect = 0;
+	return 0;
     }
     return 1;
 }
 
 
-void client_reply_to_read( struct client* client, struct nbd_request request )
+void client_reply_to_read(struct client *client,
+			  struct nbd_request request)
 {
     off64_t offset;
 
@@ -428,62 +434,74 @@ void client_reply_to_read( struct client* client, struct nbd_request request )
     /* If we get cut off partway through this sendfile, we don't
      * want to kill the server.  This should be an error.
      */
-	ERROR_IF_NEGATIVE(
+    ERROR_IF_NEGATIVE(sendfileloop(client->socket,
-			sendfileloop(
-				client->socket,
 				   client->fileno,
 				   &offset,
 				   request.len),
 		      "sendfile failed from=%ld, len=%d",
-			offset,
+		      offset, request.len);
-			request.len);
 
     sock_set_tcp_cork(client->socket, 0);
 }
 
 
-void client_reply_to_write( struct client* client, struct nbd_request request )
+void client_reply_to_write(struct client *client,
+			   struct nbd_request request)
 {
-	debug("request write from=%"PRIu64", len=%"PRIu32", handle=0x%08X", request.from, request.len, request.handle);
+    debug("request write from=%" PRIu64 ", len=%" PRIu32 ", handle=0x%08X",
+	  request.from, request.len, request.handle);
     if (client->serve->allocation_map_built) {
 	write_not_zeroes(client, request.from, request.len);
-	}
+    } else {
-	else {
 	debug("No allocation map, writing directly.");
 	/* If we get cut off partway through reading this data:
 	 * */
-		ERROR_IF_NEGATIVE(
+	ERROR_IF_NEGATIVE(readloop(client->socket,
-			readloop( client->socket,
 				   client->mapped + request.from,
 				   request.len),
 			  "reading write data failed from=%ld, len=%d",
-			request.from,
+			  request.from, request.len);
-			request.len
-		);
 
 	/* the allocation_map is shared between client threads, and may be
 	 * being built. We need to reflect the write in it, as it may be in
 	 * a position the builder has already gone over.
 	 */
-		bitset_set_range(client->serve->allocation_map, request.from, request.len);
+	bitset_set_range(client->serve->allocation_map, request.from,
+			 request.len);
     }
 
-	if (1) /* not sure whether this is necessary... */
+    // Only flush if FUA is set -- overridden for now to force flush after each
-	{
+    // write.
-		/* multiple of 4K page size */
+    // if (request.flags & CMD_FLAG_FUA) {
-		uint64_t from_rounded = request.from & (!0xfff);
+    if (1) {
+	/* multiple of page size */
+	uint64_t from_rounded =
+	    request.from & (~(sysconf(_SC_PAGE_SIZE) - 1));
 	uint64_t len_rounded = request.len + (request.from - from_rounded);
+	debug("Calling msync from=%" PRIu64 ", len=%" PRIu64 "",
+	      from_rounded, len_rounded);
 
-		FATAL_IF_NEGATIVE(
+	FATAL_IF_NEGATIVE(msync(client->mapped + from_rounded,
-			msync( client->mapped + from_rounded,
 				len_rounded,
 				MS_SYNC | MS_INVALIDATE),
-			"msync failed %ld %ld", request.from, request.len
+			  "msync failed %ld %ld", request.from,
-		);
+			  request.len);
     }
     client_write_reply(client, &request, 0);
 }
 
+void client_reply_to_flush(struct client *client,
+			   struct nbd_request request)
+{
+    debug("request flush from=%" PRIu64 ", len=%" PRIu32 ", handle=0x%08X",
+	  request.from, request.len, request.handle);
+
+    ERROR_IF_NEGATIVE(msync
+		      (client->mapped, client->mapped_size,
+		       MS_SYNC | MS_INVALIDATE), "flush failed");
+
+    client_write_reply(client, &request, 0);
+}
 
 void client_reply(struct client *client, struct nbd_request request)
 {
@@ -494,6 +512,9 @@ void client_reply( struct client* client, struct nbd_request request )
     case REQUEST_WRITE:
 	client_reply_to_write(client, request);
 	break;
+    case REQUEST_FLUSH:
+	client_reply_to_flush(client, request);
+	break;
     }
 }
 
@@ -514,8 +535,7 @@ void client_arm_killswitch( struct client* client )
 
     debug("Arming killswitch");
 
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(timer_settime(client->killswitch, 0, &its, NULL),
-		timer_settime( client->killswitch, 0, &its, NULL ),
 		      SHOW_ERRNO("Failed to arm killswitch")
 	);
 
@@ -535,8 +555,7 @@ void client_disarm_killswitch( struct client* client )
 
     debug("Disarming killswitch");
 
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(timer_settime(client->killswitch, 0, &its, NULL),
-		timer_settime( client->killswitch, 0, &its, NULL ),
 		      SHOW_ERRNO("Failed to disarm killswitch")
 	);
 
@@ -572,8 +591,9 @@ int client_serve_request(struct client* client)
     if (fd_count == 0) {
 	/* This "can't ever happen" */
 	fatal("No FDs selected, and no timeout!");
+    } else if (fd_count < 0) {
+	fatal("Select failed");
     }
-	else if ( fd_count < 0 ) { fatal( "Select failed" ); }
 
     if (self_pipe_fd_isset(client->stop_signal, &rfds)) {
 	debug("Client received stop signal.");
@@ -651,13 +671,14 @@ void client_cleanup(struct client* client,
     }
     if (client->fileno) {
 	FATAL_IF_NEGATIVE(close(client->fileno),
-			"Error closing file %d",
+			  "Error closing file %d", client->fileno);
-			client->fileno );
 	debug("Closed client file fd %d", client->fileno);
 	client->fileno = -1;
     }
 
-	if ( server_acl_locked( client->serve ) ) { server_unlock_acl( client->serve ); }
+    if (server_acl_locked(client->serve)) {
+	server_unlock_acl(client->serve);
+    }
 
 }
 
@@ -668,19 +689,18 @@ void* client_serve(void* client_uncast)
     error_set_handler((cleanup_handler *) client_cleanup, client);
 
     info("client: mmaping file");
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(open_and_mmap(client->serve->filename,
-		open_and_mmap(
-			client->serve->filename,
 				    &client->fileno,
-			NULL,
+				    &client->mapped_size,
-			(void**) &client->mapped
+				    (void **) &client->mapped),
-		),
+		      "Couldn't open/mmap file %s: %s",
-		"Couldn't open/mmap file %s: %s", client->serve->filename, strerror( errno )
+		      client->serve->filename, strerror(errno)
 	);
 
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(madvise
-		madvise( client->mapped, client->serve->size, MADV_RANDOM ),
+		      (client->mapped, client->serve->size, MADV_RANDOM),
-		SHOW_ERRNO( "Failed to madvise() %s", client->serve->filename )
+		      SHOW_ERRNO("Failed to madvise() %s",
+				 client->serve->filename)
 	);
 
     debug("Opened client file fd %d", client->fileno);
@@ -688,8 +708,7 @@ void* client_serve(void* client_uncast)
     client_send_hello(client);
 
     debug("client: serving requests");
-	while (client_serve_request(client) == 0)
+    while (client_serve_request(client) == 0);
-		;
     debug("client: stopped serving requests");
     client->stopped = 1;
 
@@ -698,10 +717,10 @@ void* client_serve(void* client_uncast)
 	server_control_arrived(client->serve);
     }
 
-	debug("Cleaning client %p up normally in thread %p", client, pthread_self());
+    debug("Cleaning client %p up normally in thread %p", client,
+	  pthread_self());
     client_cleanup(client, 0);
     debug("Client thread done");
 
     return NULL;
 }
-

src/server/client.h

@@ -3,6 +3,7 @@
 
 #include <signal.h>
 #include <time.h>
+#include <inttypes.h>
 
 /** CLIENT_HANDLER_TIMEOUT
  * This is the length of time (in seconds) any request can be outstanding for.
@@ -31,6 +32,8 @@ struct client {
     int fileno;
     char *mapped;
 
+    uint64_t mapped_size;
+
     struct self_pipe *stop_signal;
 
     struct server *serve;	/* FIXME: remove above duplication */
@@ -53,4 +56,3 @@ void client_destroy( struct client * client );
 void client_signal_stop(struct client *client);
 
 #endif
-

src/server/control.c

@@ -44,9 +44,7 @@
 #include <unistd.h>
 
 
-struct control * control_create(
+struct control *control_create(struct flexnbd *flexnbd, const char *csn)
-		struct flexnbd * flexnbd,
-		const char * csn)
 {
     struct control *control = xmalloc(sizeof(struct control));
 
@@ -80,8 +78,15 @@ void control_destroy( struct control * control )
     free(control);
 }
 
-struct control_client * control_client_create(
+void control_wait_for_close(struct control *control)
-		struct flexnbd * flexnbd,
+{
+    NULLCHECK(control);
+    while (!fd_is_closed(control->control_fd)) {
+       usleep(10000);
+    }
+}
+
+struct control_client *control_client_create(struct flexnbd *flexnbd,
 					     int client_fd,
 					     struct mbox *state_mbox)
 {
@@ -112,8 +117,7 @@ void control_handle_client( struct control * control, int client_fd )
     NULLCHECK(control);
     NULLCHECK(control->flexnbd);
     struct control_client *control_client =
-		control_client_create(
+	control_client_create(control->flexnbd,
-				control->flexnbd,
 			      client_fd,
 			      control->mirror_state_mbox);
 
@@ -132,7 +136,8 @@ void control_accept_client( struct control * control )
     union mysockaddr client_address;
     socklen_t addrlen = sizeof(union mysockaddr);
 
-	client_fd = accept( control->control_fd, &client_address.generic, &addrlen );
+    client_fd =
+	accept(control->control_fd, &client_address.generic, &addrlen);
     FATAL_IF(-1 == client_fd, "control accept failed");
 
     control_handle_client(control, client_fd);
@@ -178,25 +183,23 @@ int open_control_socket( const char * socket_name )
     }
 
     control_fd = socket(AF_UNIX, SOCK_STREAM, 0);
-	FATAL_IF_NEGATIVE(control_fd ,
+    FATAL_IF_NEGATIVE(control_fd, "Couldn't create control socket");
-	  "Couldn't create control socket");
 
     memset(&bind_address, 0, sizeof(struct sockaddr_un));
     bind_address.sun_family = AF_UNIX;
-	strncpy(bind_address.sun_path, socket_name, sizeof(bind_address.sun_path)-1);
+    strncpy(bind_address.sun_path, socket_name,
+	    sizeof(bind_address.sun_path) - 1);
 
     //unlink(socket_name); /* ignore failure */
 
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(bind
-		bind(control_fd , &bind_address, sizeof(bind_address)),
+		      (control_fd, &bind_address, sizeof(bind_address)),
 		      "Couldn't bind control socket to %s: %s",
 		      socket_name, strerror(errno)
 	);
 
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(listen(control_fd, 5),
-		listen(control_fd , 5),
+		      "Couldn't listen on control socket");
-		"Couldn't listen on control socket"
-	);
     return control_fd;
 }
 
@@ -229,8 +232,7 @@ void control_serve( struct control * control )
 }
 
 
-void control_cleanup(
+void control_cleanup(struct control *control,
-		struct control * control,
 		     int fatal __attribute__ ((unused)))
 {
     NULLCHECK(control);
@@ -256,7 +258,8 @@ void * control_runner( void * control_uncast )
 
 #define write_socket(msg) write(client_fd, (msg "\n"), strlen((msg))+1)
 
-void control_write_mirror_response( enum mirror_state mirror_state, int client_fd )
+void control_write_mirror_response(enum mirror_state mirror_state,
+				   int client_fd)
 {
     switch (mirror_state) {
     case MS_INIT:
@@ -286,12 +289,12 @@ void control_write_mirror_response( enum mirror_state mirror_state, int client_f
 	fatal("Unhandled mirror state: %d", mirror_state);
     }
 }
+
 #undef write_socket
 
 
 /* Call this in the thread where you want to receive the mirror state */
-enum mirror_state control_client_mirror_wait(
+enum mirror_state control_client_mirror_wait(struct control_client *client)
-		struct control_client* client)
 {
     NULLCHECK(client);
     NULLCHECK(client->mirror_state_mbox);
@@ -345,14 +348,11 @@ int control_mirror(struct control_client* client, int linesc, char** lines)
     if (linesc > 2) {
 	if (strcmp("exit", lines[2]) == 0) {
 	    action_at_finish = ACTION_EXIT;
-		}
+	} else if (strcmp("unlink", lines[2]) == 0) {
-		else if (strcmp( "unlink", lines[2]) == 0 ) {
 	    action_at_finish = ACTION_UNLINK;
-		}
+	} else if (strcmp("nothing", lines[2]) == 0) {
-		else if (strcmp("nothing", lines[2]) == 0) {
 	    action_at_finish = ACTION_NOTHING;
-		}
+	} else {
-		else {
 	    write_socket("1: action must be 'exit' or 'nothing'");
 	    return -1;
 	}
@@ -389,13 +389,13 @@ int control_mirror(struct control_client* client, int linesc, char** lines)
     server_lock_start_mirror(serve);
     {
 	if (server_mirror_can_start(serve)) {
-			serve->mirror_super = mirror_super_create(
+	    serve->mirror_super = mirror_super_create(serve->filename,
-					serve->filename,
 						      connect_to,
 						      connect_from,
 						      max_Bps,
 						      action_at_finish,
-					client->mirror_state_mbox );
+						      client->
+						      mirror_state_mbox);
 	    serve->mirror = serve->mirror_super->mirror;
 	    server_prevent_mirror_start(serve);
 	} else {
@@ -415,18 +415,14 @@ int control_mirror(struct control_client* client, int linesc, char** lines)
      * sighandler can block the serve thread
      */
     if (serve->mirror_super) {
-		FATAL_IF( 0 != pthread_create(
+	FATAL_IF(0 != pthread_create(&serve->mirror_super->thread,
-					&serve->mirror_super->thread,
 				     NULL,
 				     mirror_super_runner,
-					serve
+				     serve),
-					),
+		 "Failed to create mirror thread");
-				"Failed to create mirror thread"
-			);
 
 	debug("Control thread mirror super waiting");
-		enum mirror_state state =
+	enum mirror_state state = control_client_mirror_wait(client);
-			control_client_mirror_wait( client );
 	debug("Control thread writing response");
 	control_write_mirror_response(state, client->socket);
     }
@@ -436,7 +432,8 @@ int control_mirror(struct control_client* client, int linesc, char** lines)
     return 0;
 }
 
-int control_mirror_max_bps( struct control_client* client, int linesc, char** lines )
+int control_mirror_max_bps(struct control_client *client, int linesc,
+			   char **lines)
 {
     NULLCHECK(client);
     NULLCHECK(client->flexnbd);
@@ -489,8 +486,7 @@ int control_acl(struct control_client* client, int linesc, char** lines)
 	      strlen(lines[new_acl->len]));
 	write(client->socket, "\n", 1);
 	acl_destroy(new_acl);
-	}
+    } else {
-	else {
 	flexnbd_replace_acl(flexnbd, new_acl);
 	info("ACL set");
 	write(client->socket, "0: updated\n", 11);
@@ -500,8 +496,7 @@ int control_acl(struct control_client* client, int linesc, char** lines)
 }
 
 
-int control_break(
+int control_break(struct control_client *client,
-		struct control_client* client,
 		  int linesc __attribute__ ((unused)),
 		  char **lines __attribute__ ((unused))
     )
@@ -524,13 +519,10 @@ int control_break(
 
 	    if (server_is_closed(serve)) {
 		info("Mirror completed while canceling");
-				write( client->socket,
+		write(client->socket, "1: mirror completed\n", 20);
-						"1: mirror completed\n", 20 );
+	    } else {
-			}
-			else {
 		info("Mirror successfully stopped.");
-				write( client->socket,
+		write(client->socket, "0: mirror stopped\n", 18);
-						"0: mirror stopped\n", 18 );
 		result = 1;
 	    }
 
@@ -546,8 +538,7 @@ int control_break(
 
 
 /** FIXME: add some useful statistics */
-int control_status(
+int control_status(struct control_client *client,
-		struct control_client* client,
 		   int linesc __attribute__ ((unused)),
 		   char **lines __attribute__ ((unused))
     )
@@ -566,10 +557,14 @@ int control_status(
 void control_client_cleanup(struct control_client *client,
 			    int fatal __attribute__ ((unused)))
 {
-	if (client->socket) { close(client->socket); }
+    if (client->socket) {
+	close(client->socket);
+    }
 
     /* This is wrongness */
-	if ( server_acl_locked( client->flexnbd->serve ) ) { server_unlock_acl( client->flexnbd->serve ); }
+    if (server_acl_locked(client->flexnbd->serve)) {
+	server_unlock_acl(client->flexnbd->serve);
+    }
 
     control_client_destroy(client);
 }
@@ -584,30 +579,25 @@ void control_respond(struct control_client * client)
     int i, linesc;
     linesc = read_lines_until_blankline(client->socket, 256, &lines);
 
-	if (linesc < 1)
+    if (linesc < 1) {
-	{
 	write(client->socket, "9: missing command\n", 19);
 	/* ignore failure */
-	}
+    } else if (strcmp(lines[0], "acl") == 0) {
-	else if (strcmp(lines[0], "acl") == 0) {
 	info("acl command received");
 	if (control_acl(client, linesc - 1, lines + 1) < 0) {
 	    debug("acl command failed");
 	}
-	}
+    } else if (strcmp(lines[0], "mirror") == 0) {
-	else if (strcmp(lines[0], "mirror") == 0) {
 	info("mirror command received");
 	if (control_mirror(client, linesc - 1, lines + 1) < 0) {
 	    debug("mirror command failed");
 	}
-	}
+    } else if (strcmp(lines[0], "break") == 0) {
-	else if (strcmp(lines[0], "break") == 0) {
 	info("break command received");
 	if (control_break(client, linesc - 1, lines + 1) < 0) {
 	    debug("break command failed");
 	}
-	}
+    } else if (strcmp(lines[0], "status") == 0) {
-	else if (strcmp(lines[0], "status") == 0) {
 	info("status command received");
 	if (control_status(client, linesc - 1, lines + 1) < 0) {
 	    debug("status command failed");
@@ -617,8 +607,7 @@ void control_respond(struct control_client * client)
 	if (control_mirror_max_bps(client, linesc - 1, lines + 1) < 0) {
 	    debug("mirror_max_bps command failed");
 	}
-	}
+    } else {
-	else {
 	write(client->socket, "10: unknown command\n", 23);
     }
 
@@ -630,4 +619,3 @@ void control_respond(struct control_client * client)
     control_client_cleanup(client, 0);
     debug("control command handled");
 }
-

src/server/control.h

@@ -44,16 +44,16 @@ struct control_client{
     struct mbox *mirror_state_mbox;
 };
 
-struct control * control_create(
+struct control *control_create(struct flexnbd *,
-		struct flexnbd *,
 			       const char *control_socket_name);
 void control_signal_close(struct control *);
+void control_wait_for_close(struct control *control);
 void control_destroy(struct control *);
 
 void *control_runner(void *);
 
-void accept_control_connection(struct server* params, int client_fd, union mysockaddr* client_address);
+void accept_control_connection(struct server *params, int client_fd,
+			       union mysockaddr *client_address);
 void serve_open_control_socket(struct server *params);
 
 #endif
-

src/server/flexnbd.c

@@ -61,16 +61,13 @@ int flexnbd_build_signal_fd(void)
 }
 
 
-void flexnbd_create_shared(
+void flexnbd_create_shared(struct flexnbd *flexnbd,
-	struct flexnbd * flexnbd,
 			   const char *s_ctrl_sock)
 {
     NULLCHECK(flexnbd);
     if (s_ctrl_sock) {
-		flexnbd->control =
+	flexnbd->control = control_create(flexnbd, s_ctrl_sock);
-			control_create( flexnbd, s_ctrl_sock );
+    } else {
-	}
-	else {
 	flexnbd->control = NULL;
     }
 
@@ -78,8 +75,7 @@ void flexnbd_create_shared(
 }
 
 
-struct flexnbd * flexnbd_create_serving(
+struct flexnbd *flexnbd_create_serving(char *s_ip_address,
-	char* s_ip_address,
 				       char *s_port,
 				       char *s_file,
 				       char *s_ctrl_sock,
@@ -90,17 +86,14 @@ struct flexnbd * flexnbd_create_serving(
 				       int use_killswitch)
 {
     struct flexnbd *flexnbd = xmalloc(sizeof(struct flexnbd));
-	flexnbd->serve = server_create(
+    flexnbd->serve = server_create(flexnbd,
-			flexnbd,
 				   s_ip_address,
 				   s_port,
 				   s_file,
 				   default_deny,
 				   acl_entries,
 				   s_acl_entries,
-			max_nbd_clients,
+				   max_nbd_clients, use_killswitch, 1);
-			use_killswitch,
-			1);
     flexnbd_create_shared(flexnbd, s_ctrl_sock);
 
     // Beats installing one handler per client instance
@@ -110,17 +103,14 @@ struct flexnbd * flexnbd_create_serving(
 	    .sa_flags = SA_RESTART | SA_SIGINFO
 	};
 
-		FATAL_UNLESS(
+	FATAL_UNLESS(0 == sigaction(CLIENT_KILLSWITCH_SIGNAL, &act, NULL),
-			0 == sigaction( CLIENT_KILLSWITCH_SIGNAL, &act, NULL ),
+		     "Installing client killswitch signal failed");
-			"Installing client killswitch signal failed"
-		);
     }
 
     return flexnbd;
 }
 
-struct flexnbd * flexnbd_create_listening(
+struct flexnbd *flexnbd_create_listening(char *s_ip_address,
-		char* s_ip_address,
 					 char *s_port,
 					 char *s_file,
 					 char *s_ctrl_sock,
@@ -129,15 +119,12 @@ struct flexnbd * flexnbd_create_listening(
 					 char **s_acl_entries)
 {
     struct flexnbd *flexnbd = xmalloc(sizeof(struct flexnbd));
-	flexnbd->serve = server_create(
+    flexnbd->serve = server_create(flexnbd,
-			flexnbd,
 				   s_ip_address,
 				   s_port,
 				   s_file,
 				   default_deny,
-			acl_entries,
+				   acl_entries, s_acl_entries, 1, 0, 0);
-			s_acl_entries,
-			1, 0, 0);
     flexnbd_create_shared(flexnbd, s_ctrl_sock);
 
     // listen can't use killswitch, as mirror may pause on sending things
@@ -153,8 +140,7 @@ void flexnbd_spawn_control(struct flexnbd * flexnbd )
 
     pthread_t *control_thread = &flexnbd->control->thread;
 
-	FATAL_UNLESS( 0 == pthread_create(
+    FATAL_UNLESS(0 == pthread_create(control_thread,
-				control_thread,
 				     NULL,
 				     control_runner,
 				     flexnbd->control),
@@ -233,9 +219,7 @@ void make_writable( const char * filename )
 {
     NULLCHECK(filename);
     FATAL_IF_NEGATIVE(chmod(filename, S_IWUSR),
-			"Couldn't chmod %s: %s",
+		      "Couldn't chmod %s: %s", filename, strerror(errno));
-			filename,
-			strerror( errno ) );
 }
 
 
@@ -262,4 +246,3 @@ int flexnbd_serve( struct flexnbd * flexnbd )
 
     return success;
 }
-

src/server/flexnbd.h

@@ -29,8 +29,7 @@ struct flexnbd {
 
 
 struct flexnbd *flexnbd_create(void);
-struct flexnbd * flexnbd_create_serving(
+struct flexnbd *flexnbd_create_serving(char *s_ip_address,
-	char* s_ip_address,
 				       char *s_port,
 				       char *s_file,
 				       char *s_ctrl_sock,
@@ -40,8 +39,7 @@ struct flexnbd * flexnbd_create_serving(
 				       int max_nbd_clients,
 				       int use_killswitch);
 
-struct flexnbd * flexnbd_create_listening(
+struct flexnbd *flexnbd_create_listening(char *s_ip_address,
-	char* s_ip_address,
 					 char *s_port,
 					 char *s_file,
 					 char *s_ctrl_sock,
@@ -63,4 +61,3 @@ struct server * flexnbd_server( struct flexnbd * flexnbd );
 void flexnbd_replace_acl(struct flexnbd *flexnbd, struct acl *acl);
 struct status *flexnbd_status_create(struct flexnbd *flexnbd);
 #endif
-

src/server/flexthread.c

@@ -22,15 +22,14 @@ void flexthread_mutex_destroy( struct flexthread_mutex * ftm )
 
     if (flexthread_mutex_held(ftm)) {
 	flexthread_mutex_unlock(ftm);
-	}
+    } else if ((pthread_t) NULL != ftm->holder) {
-	else if ( (pthread_t)NULL != ftm->holder ) {
 	/* This "should never happen": if we can try to destroy
 	 * a mutex currently held by another thread, there's a
 	 * logic bug somewhere.  I know the test here is racy,
 	 * but there's not a lot we can do about it at this
 	 * point.
 	 */
-		fatal( "Attempted to destroy a flexthread_mutex"\
+	fatal("Attempted to destroy a flexthread_mutex"
 	      " held by another thread!");
     }
 
@@ -72,4 +71,3 @@ int flexthread_mutex_held( struct flexthread_mutex * ftm )
     NULLCHECK(ftm);
     return pthread_self() == ftm->holder;
 }
-

src/server/mirror.c

@@ -90,8 +90,7 @@ struct mirror_ctrl {
 
 };
 
-struct mirror * mirror_alloc(
+struct mirror *mirror_alloc(union mysockaddr *connect_to,
-		union mysockaddr * connect_to,
 			    union mysockaddr *connect_from,
 			    uint64_t max_Bps,
 			    enum mirror_finish_action action_at_finish,
@@ -144,19 +143,13 @@ void mirror_init( struct mirror * mirror, const char * filename )
     NULLCHECK(mirror);
     NULLCHECK(filename);
 
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(open_and_mmap(filename,
-		open_and_mmap(
-			filename,
 				    &map_fd,
 				    &size,
-			(void**) &mirror->mapped
+				    (void **) &mirror->mapped),
-		),
+		      "Failed to open and mmap %s", filename);
-		"Failed to open and mmap %s",
-		filename
-	);
 
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(madvise(mirror->mapped, size, MADV_SEQUENTIAL),
-		madvise( mirror->mapped, size, MADV_SEQUENTIAL ),
 		      SHOW_ERRNO("Failed to madvise() %s", filename)
 	);
 }
@@ -176,8 +169,7 @@ void mirror_reset( struct mirror * mirror )
 }
 
 
-struct mirror * mirror_create(
+struct mirror *mirror_create(const char *filename,
-		const char * filename,
 			     union mysockaddr *connect_to,
 			     union mysockaddr *connect_from,
 			     uint64_t max_Bps,
@@ -189,9 +181,7 @@ struct mirror * mirror_create(
 
     mirror = mirror_alloc(connect_to,
 			  connect_from,
-			max_Bps,
+			  max_Bps, action_at_finish, commit_signal);
-			action_at_finish,
-			commit_signal);
 
     mirror_init(mirror, filename);
     mirror_reset(mirror);
@@ -281,7 +271,8 @@ int mirror_connect( struct mirror * mirror, uint64_t local_size )
 
     NULLCHECK(mirror->connect_to);
 
-	mirror->client = socket_connect(&mirror->connect_to->generic, connect_from);
+    mirror->client =
+	socket_connect(&mirror->connect_to->generic, connect_from);
     if (0 < mirror->client) {
 	fd_set fds;
 	struct timeval tv = { MS_HELLO_TIME_SECS, 0 };
@@ -293,30 +284,30 @@ int mirror_connect( struct mirror * mirror, uint64_t local_size )
 
 	if (FD_ISSET(mirror->client, &fds)) {
 	    uint64_t remote_size;
-			if ( socket_nbd_read_hello( mirror->client, &remote_size ) ) {
+	    uint32_t remote_flags;
+	    if (socket_nbd_read_hello
+		(mirror->client, &remote_size, &remote_flags)) {
 		if (remote_size == local_size) {
 		    connected = 1;
 		    mirror_set_state(mirror, MS_GO);
-				}
+		} else {
-				else {
 		    warn("Remote size (%d) doesn't match local (%d)",
 			 remote_size, local_size);
 		    mirror_set_state(mirror, MS_FAIL_SIZE_MISMATCH);
 		}
-			}
+	    } else {
-			else {
 		warn("Mirror attempt rejected.");
 		mirror_set_state(mirror, MS_FAIL_REJECTED);
 	    }
-		}
+	} else {
-		else {
 	    warn("No NBD Hello received.");
 	    mirror_set_state(mirror, MS_FAIL_NO_HELLO);
 	}
 
-		if ( !connected ) { close( mirror->client ); }
+	if (!connected) {
+	    close(mirror->client);
 	}
-	else {
+    } else {
 	warn("Mirror failed to connect.");
 	mirror_set_state(mirror, MS_FAIL_CONNECT);
     }
@@ -371,12 +362,15 @@ int mirror_setup_next_xfer( struct mirror_ctrl *ctrl )
      * full, and stop when it's a quarter full. This stops a busy client from
      * stalling a migration forever. FIXME: made-up numbers.
      */
-	if ( mirror->offset < serve->size && bitset_stream_size( serve->allocation_map ) > BITSET_STREAM_SIZE / 2 ) {
+    if (mirror->offset < serve->size
+	&& bitset_stream_size(serve->allocation_map) >
+	BITSET_STREAM_SIZE / 2) {
 	ctrl->clear_events = 1;
     }
 
 
-	while ( ( mirror->offset == serve->size || ctrl->clear_events ) && e.event != BITSET_STREAM_SET ) {
+    while ((mirror->offset == serve->size || ctrl->clear_events)
+	   && e.event != BITSET_STREAM_SET) {
 	uint64_t events = bitset_stream_size(serve->allocation_map);
 
 	if (events == 0) {
@@ -408,7 +402,8 @@ int mirror_setup_next_xfer( struct mirror_ctrl *ctrl )
 	return 0;
     }
 
-	debug( "Next transfer: current=%"PRIu64", run=%"PRIu64, current, run );
+    debug("Next transfer: current=%" PRIu64 ", run=%" PRIu64, current,
+	  run);
     struct nbd_request req = {
 	.magic = REQUEST_MAGIC,
 	.type = REQUEST_WRITE,
@@ -460,7 +455,8 @@ static void mirror_write_cb( struct ev_loop *loop, ev_io *w, int revents )
 	return;
     }
 
-	debug( "Mirror write callback invoked with events %d. fd: %i", revents, ctrl->mirror->client );
+    debug("Mirror write callback invoked with events %d. fd: %i", revents,
+	  ctrl->mirror->client);
 
     /* FIXME: We can end up corking multiple times in unusual circumstances; this
      * is annoying, but harmless */
@@ -472,7 +468,8 @@ static void mirror_write_cb( struct ev_loop *loop, ev_io *w, int revents )
 	data_loc = ((char *) &xfer->hdr.req_raw) + ctrl->xfer.written;
 	to_write = hdr_size - xfer->written;
     } else {
-		data_loc = ctrl->mirror->mapped  + xfer->from + ( xfer->written - hdr_size );
+	data_loc =
+	    ctrl->mirror->mapped + xfer->from + (xfer->written - hdr_size);
 	to_write = xfer->len - (ctrl->xfer.written - hdr_size);
     }
 
@@ -485,14 +482,14 @@ static void mirror_write_cb( struct ev_loop *loop, ev_io *w, int revents )
 	return;
     }
     debug("Wrote %" PRIu64 " bytes", count);
-	debug( "to_write was %"PRIu64", xfer->written was %"PRIu64, to_write, xfer->written );
+    debug("to_write was %" PRIu64 ", xfer->written was %" PRIu64, to_write,
+	  xfer->written);
 
     // We wrote some bytes, so reset the timer and keep track for the next pass
     if (count > 0) {
 	ctrl->xfer.written += count;
 	ev_timer_again(ctrl->ev_loop, &ctrl->timeout_watcher);
     }
-
     // All bytes written, so now we need to read the NBD reply back.
     if (ctrl->xfer.written == ctrl->xfer.len + hdr_size) {
 	sock_set_tcp_cork(ctrl->mirror->client, 0);
@@ -523,10 +520,13 @@ static void mirror_read_cb( struct ev_loop *loop, ev_io *w, int revents )
     ssize_t count;
     uint64_t left = sizeof(struct nbd_reply_raw) - xfer->read;
 
-	debug( "Mirror read callback invoked with events %d. fd:%i", revents, m->client );
+    debug("Mirror read callback invoked with events %d. fd:%i", revents,
+	  m->client);
 
     /* Start / continue reading the NBD response from the mirror. */
-	if ( ( count = read( m->client, ((void*) &xfer->hdr.rsp_raw) + xfer->read, left ) ) < 0 ) {
+    if ((count =
+	 read(m->client, ((void *) &xfer->hdr.rsp_raw) + xfer->read,
+	      left)) < 0) {
 	if (errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR) {
 	    warn(SHOW_ERRNO("Couldn't read from listener"));
 	    ev_break(loop, EVBREAK_ONE);
@@ -540,12 +540,12 @@ static void mirror_read_cb( struct ev_loop *loop, ev_io *w, int revents )
 	ev_break(loop, EVBREAK_ONE);
 	return;
     }
-
     // We read some bytes, so reset the timer
     ev_timer_again(ctrl->ev_loop, &ctrl->timeout_watcher);
 
     debug("Read %i bytes", count);
-	debug( "left was %"PRIu64", xfer->read was %"PRIu64, left, xfer->read );
+    debug("left was %" PRIu64 ", xfer->read was %" PRIu64, left,
+	  xfer->read);
     xfer->read += count;
 
     if (xfer->read < sizeof(struct nbd_reply_raw)) {
@@ -602,7 +602,9 @@ static void mirror_read_cb( struct ev_loop *loop, ev_io *w, int revents )
     debug("next_xfer: %d", next_xfer);
 
     /* Regardless of time estimates, if there's no waiting transfer, we can start closing clients down. */
-	if ( !ctrl->clients_closed && ( !next_xfer || server_mirror_eta( ctrl->serve ) < MS_CONVERGE_TIME_SECS ) ) {
+    if (!ctrl->clients_closed
+	&& (!next_xfer
+	    || server_mirror_eta(ctrl->serve) < MS_CONVERGE_TIME_SECS)) {
 	info("Closing clients to allow mirroring to converge");
 	server_forbid_new_clients(ctrl->serve);
 	server_close_clients(ctrl->serve);
@@ -643,7 +645,8 @@ static void mirror_read_cb( struct ev_loop *loop, ev_io *w, int revents )
     return;
 }
 
-static void mirror_timeout_cb( struct ev_loop *loop, ev_timer *w __attribute__((unused)), int revents )
+static void mirror_timeout_cb(struct ev_loop *loop, ev_timer * w
+			      __attribute__ ((unused)), int revents)
 {
     if (!(revents & EV_TIMER)) {
 	warn("Mirror timeout called but no timer event signalled");
@@ -668,12 +671,14 @@ static void mirror_abandon_cb( struct ev_loop *loop, ev_io *w, int revents )
     debug("Abandon message received");
     mirror_set_state(ctrl->mirror, MS_ABANDONED);
     self_pipe_signal_clear(ctrl->mirror->abandon_signal);
+    ev_io_stop(loop, &ctrl->abandon_watcher);
     ev_break(loop, EVBREAK_ONE);
     return;
 }
 
 
-static void mirror_limit_cb( struct ev_loop *loop, ev_timer *w, int revents )
+static void mirror_limit_cb(struct ev_loop *loop, ev_timer * w,
+			    int revents)
 {
     struct mirror_ctrl *ctrl = (struct mirror_ctrl *) w->data;
     NULLCHECK(ctrl);
@@ -684,7 +689,8 @@ static void mirror_limit_cb( struct ev_loop *loop, ev_timer *w, int revents )
     }
 
     if (mirror_should_wait(ctrl)) {
-		debug( "max_bps exceeded, waiting", ctrl->mirror->max_bytes_per_second );
+	debug("max_bps exceeded, waiting",
+	      ctrl->mirror->max_bytes_per_second);
 	ev_timer_again(loop, w);
     } else {
 	/* We're below the limit, so do the next request */
@@ -701,7 +707,8 @@ static void mirror_limit_cb( struct ev_loop *loop, ev_timer *w, int revents )
  * if it has, start migrating. If it's not finished, then enabling the bitset
  * stream does not go well for us.
  */
-static void mirror_begin_cb( struct ev_loop *loop, ev_timer *w, int revents )
+static void mirror_begin_cb(struct ev_loop *loop, ev_timer * w,
+			    int revents)
 {
     struct mirror_ctrl *ctrl = (struct mirror_ctrl *) w->data;
     NULLCHECK(ctrl);
@@ -711,7 +718,8 @@ static void mirror_begin_cb( struct ev_loop *loop, ev_timer *w, int revents )
 	return;
     }
 
-	if ( ctrl->serve->allocation_map_built || ctrl->serve->allocation_map_not_built ) {
+    if (ctrl->serve->allocation_map_built
+	|| ctrl->serve->allocation_map_not_built) {
 	info("allocation map builder is finished, beginning migration");
 	ev_timer_stop(loop, w);
 	/* Start by writing xfer 0 to the listener */
@@ -796,10 +804,8 @@ void mirror_run( struct server *serve )
     ctrl.abandon_watcher.data = (void *) &ctrl;
     ev_io_start(ctrl.ev_loop, &ctrl.abandon_watcher);
 
-	ERROR_UNLESS(
+    ERROR_UNLESS(mirror_setup_next_xfer(&ctrl),
-		mirror_setup_next_xfer( &ctrl ),
+		 "Couldn't find first transfer for mirror!");
-		"Couldn't find first transfer for mirror!"
-	);
 
 
     if (serve->allocation_map_built) {
@@ -830,7 +836,8 @@ void mirror_run( struct server *serve )
      * it to something sane - they just terminate the event loop with state !=
      * MS_DONE. We re-allow new clients here if necessary.
      */
-	if ( m->action_at_finish == ACTION_NOTHING || m->commit_state != MS_DONE ) {
+    if (m->action_at_finish == ACTION_NOTHING
+	|| m->commit_state != MS_DONE) {
 	server_allow_new_clients(serve);
     }
 
@@ -896,7 +903,9 @@ void* mirror_runner(void* serve_params_uncast)
     time_t start_time = time(NULL);
     int connected = mirror_connect(mirror, serve->size);
     mirror_signal_commit(mirror);
-	if ( !connected ) { goto abandon_mirror; }
+    if (!connected) {
+	goto abandon_mirror;
+    }
 
     /* After this point, if we see a failure we need to disconnect
      * and retry everything from mirror_set_state(_, MS_INIT), but
@@ -932,38 +941,33 @@ abandon_mirror:
 }
 
 
-struct mirror_super * mirror_super_create(
+struct mirror_super *mirror_super_create(const char *filename,
-		const char * filename,
 					 union mysockaddr *connect_to,
 					 union mysockaddr *connect_from,
 					 uint64_t max_Bps,
-		enum mirror_finish_action action_at_finish,
+					 enum mirror_finish_action
+					 action_at_finish,
 					 struct mbox *state_mbox)
 {
     struct mirror_super *super = xmalloc(sizeof(struct mirror_super));
-	super->mirror = mirror_create(
+    super->mirror = mirror_create(filename,
-			filename,
 				  connect_to,
 				  connect_from,
 				  max_Bps,
-			action_at_finish,
+				  action_at_finish, mbox_create());
-			mbox_create() ) ;
     super->state_mbox = state_mbox;
     return super;
 }
 
 
 /* Post the current state of the mirror into super->state_mbox.*/
-void mirror_super_signal_committed(
+void mirror_super_signal_committed(struct mirror_super *super,
-		struct mirror_super * super ,
 				   enum mirror_state commit_state)
 {
     NULLCHECK(super);
     NULLCHECK(super->state_mbox);
 
-	mbox_post_mirror_state(
+    mbox_post_mirror_state(super->state_mbox, commit_state);
-			super->state_mbox,
-			commit_state );
 }
 
 
@@ -997,8 +1001,7 @@ void * mirror_super_runner( void * serve_uncast )
     struct mirror_super *super = serve->mirror_super;
 
     do {
-		FATAL_IF( 0 != pthread_create(
+	FATAL_IF(0 != pthread_create(&mirror->thread,
-					&mirror->thread,
 				     NULL,
 				     mirror_runner,
 				     serve),
@@ -1018,9 +1021,7 @@ void * mirror_super_runner( void * serve_uncast )
 	    should_retry = *commit_state == MS_GO;
 
 	    /* Only send this signal the first time */
-			mirror_super_signal_committed(
+	    mirror_super_signal_committed(super, *commit_state);
-					super,
-					*commit_state);
 	    debug("Mirror supervisor committed");
 	}
 	/* We only care about the value of the commit signal on

src/server/mirror.h

@@ -127,15 +127,13 @@ struct mirror_super {
 struct server;
 struct flexnbd;
 
-struct mirror_super * mirror_super_create(
+struct mirror_super *mirror_super_create(const char *filename,
-		const char * filename,
 					 union mysockaddr *connect_to,
 					 union mysockaddr *connect_from,
 					 uint64_t max_Bps,
-		enum mirror_finish_action action_at_finish,
+					 enum mirror_finish_action
-		struct mbox * state_mbox
+					 action_at_finish,
-		);
+					 struct mbox *state_mbox);
 void *mirror_super_runner(void *serve_uncast);
 
 #endif
-

src/server/mode.c

@@ -19,6 +19,7 @@ static struct option serve_options[] = {
     GETOPT_VERBOSE,
     {0}
 };
+
 static char serve_short_options[] = "hl:p:f:s:dk" SOPT_QUIET SOPT_VERBOSE;
 static char serve_help_text[] =
     "Usage: flexnbd " CMD_SERVE " <options> [<acl address>*]\n\n"
@@ -28,10 +29,9 @@ static char serve_help_text[] =
     "\t--" OPT_PORT ",-p <PORT>\tThe port to serve on.\n"
     "\t--" OPT_FILE ",-f <FILE>\tThe file to serve.\n"
     "\t--" OPT_DENY ",-d\tDeny connections by default unless in ACL.\n"
-	"\t--" OPT_KILLSWITCH",-k  \tKill the server if a request takes 120 seconds.\n"
+    "\t--" OPT_KILLSWITCH
-	SOCK_LINE
+    ",-k  \tKill the server if a request takes 120 seconds.\n" SOCK_LINE
-	VERBOSE_LINE
+    VERBOSE_LINE QUIET_LINE;
-	QUIET_LINE;
 
 
 static struct option listen_options[] = {
@@ -45,6 +45,7 @@ static struct option listen_options[] = {
     GETOPT_VERBOSE,
     {0}
 };
+
 static char listen_short_options[] = "hl:p:f:s:d" SOPT_QUIET SOPT_VERBOSE;
 static char listen_help_text[] =
     "Usage: flexnbd " CMD_LISTEN " <options> [<acl_address>*]\n\n"
@@ -54,9 +55,7 @@ static char listen_help_text[] =
     "\t--" OPT_PORT ",-p <PORT>\tThe port to listen on.\n"
     "\t--" OPT_FILE ",-f <FILE>\tThe file to serve.\n"
     "\t--" OPT_DENY ",-d\tDeny connections by default unless in ACL.\n"
-	SOCK_LINE
+    SOCK_LINE VERBOSE_LINE QUIET_LINE;
-	VERBOSE_LINE
-	QUIET_LINE;
 
 static struct option read_options[] = {
     GETOPT_HELP,
@@ -69,6 +68,7 @@ static struct option read_options[] = {
     GETOPT_VERBOSE,
     {0}
 };
+
 static char read_short_options[] = "hl:p:F:S:b:" SOPT_QUIET SOPT_VERBOSE;
 static char read_help_text[] =
     "Usage: flexnbd " CMD_READ " <options>\n\n"
@@ -78,9 +78,7 @@ static char read_help_text[] =
     "\t--" OPT_PORT ",-p <PORT>\tThe port to read from.\n"
     "\t--" OPT_FROM ",-F <OFFSET>\tByte offset to read from.\n"
     "\t--" OPT_SIZE ",-S <SIZE>\tBytes to read.\n"
-	BIND_LINE
+    BIND_LINE VERBOSE_LINE QUIET_LINE;
-	VERBOSE_LINE
-	QUIET_LINE;
 
 
 static struct option *write_options = read_options;
@@ -93,9 +91,7 @@ static char write_help_text[] =
     "\t--" OPT_PORT ",-p <PORT>\tThe port to write to.\n"
     "\t--" OPT_FROM ",-F <OFFSET>\tByte offset to write from.\n"
     "\t--" OPT_SIZE ",-S <SIZE>\tBytes to write.\n"
-	BIND_LINE
+    BIND_LINE VERBOSE_LINE QUIET_LINE;
-	VERBOSE_LINE
-	QUIET_LINE;
 
 static struct option acl_options[] = {
     GETOPT_HELP,
@@ -104,14 +100,12 @@ static struct option acl_options[] = {
     GETOPT_VERBOSE,
     {0}
 };
+
 static char acl_short_options[] = "hs:" SOPT_QUIET SOPT_VERBOSE;
 static char acl_help_text[] =
     "Usage: flexnbd " CMD_ACL " <options> [<acl address>+]\n\n"
     "Set the access control list for a server with control socket SOCK.\n\n"
-	HELP_LINE
+    HELP_LINE SOCK_LINE VERBOSE_LINE QUIET_LINE;
-	SOCK_LINE
-	VERBOSE_LINE
-	QUIET_LINE;
 
 static struct option mirror_speed_options[] = {
     GETOPT_HELP,
@@ -121,15 +115,12 @@ static struct option mirror_speed_options[] = {
     GETOPT_VERBOSE,
     {0}
 };
+
 static char mirror_speed_short_options[] = "hs:m:" SOPT_QUIET SOPT_VERBOSE;
 static char mirror_speed_help_text[] =
     "Usage: flexnbd " CMD_MIRROR_SPEED " <options>\n\n"
     "Set the maximum speed of a migration from a mirring server listening on SOCK.\n\n"
-	HELP_LINE
+    HELP_LINE SOCK_LINE MAX_SPEED_LINE VERBOSE_LINE QUIET_LINE;
-	SOCK_LINE
-	MAX_SPEED_LINE
-	VERBOSE_LINE
-	QUIET_LINE;
 
 static struct option mirror_options[] = {
     GETOPT_HELP,
@@ -142,6 +133,7 @@ static struct option mirror_options[] = {
     GETOPT_VERBOSE,
     {0}
 };
+
 static char mirror_short_options[] = "hs:l:p:ub:" SOPT_QUIET SOPT_VERBOSE;
 static char mirror_help_text[] =
     "Usage: flexnbd " CMD_MIRROR " <options>\n\n"
@@ -151,9 +143,7 @@ static char mirror_help_text[] =
     "\t--" OPT_PORT ",-p <PORT>\tThe port to mirror to.\n"
     SOCK_LINE
     "\t--" OPT_UNLINK ",-u\tUnlink the local file when done.\n"
-	BIND_LINE
+    BIND_LINE VERBOSE_LINE QUIET_LINE;
-	VERBOSE_LINE
-	QUIET_LINE;
 
 static struct option break_options[] = {
     GETOPT_HELP,
@@ -162,14 +152,12 @@ static struct option break_options[] = {
     GETOPT_VERBOSE,
     {0}
 };
+
 static char break_short_options[] = "hs:" SOPT_QUIET SOPT_VERBOSE;
 static char break_help_text[] =
     "Usage: flexnbd " CMD_BREAK " <options>\n\n"
     "Stop mirroring from the server with control socket SOCK.\n\n"
-	HELP_LINE
+    HELP_LINE SOCK_LINE VERBOSE_LINE QUIET_LINE;
-	SOCK_LINE
-	VERBOSE_LINE
-	QUIET_LINE;
 
 
 static struct option status_options[] = {
@@ -179,14 +167,12 @@ static struct option status_options[] = {
     GETOPT_VERBOSE,
     {0}
 };
+
 static char status_short_options[] = "hs:" SOPT_QUIET SOPT_VERBOSE;
 static char status_help_text[] =
     "Usage: flexnbd " CMD_STATUS " <options>\n\n"
     "Get the status for a server with control socket SOCK.\n\n"
-	HELP_LINE
+    HELP_LINE SOCK_LINE VERBOSE_LINE QUIET_LINE;
-	SOCK_LINE
-	VERBOSE_LINE
-	QUIET_LINE;
 
 char help_help_text_arr[] =
     "Usage: flexnbd <cmd> [cmd options]\n\n"
@@ -200,8 +186,7 @@ char help_help_text_arr[] =
     "\tflexnbd mirror-speed\n"
     "\tflexnbd break\n"
     "\tflexnbd status\n"
-	"\tflexnbd help\n\n"
+    "\tflexnbd help\n\n" "See flexnbd help <cmd> for further info\n";
-	"See flexnbd help <cmd> for further info\n";
 /* Slightly odd array/pointer pair to stop the compiler from complaining
  * about symbol sizes
  */
@@ -214,7 +199,8 @@ void do_write(struct mode_readwrite_params* params);
 void do_remote_command(char *command, char *mode, int argc, char **argv);
 
 
-void read_serve_param( int c, char **ip_addr, char **ip_port, char **file, char **sock, int *default_deny, int *use_killswitch )
+void read_serve_param(int c, char **ip_addr, char **ip_port, char **file,
+		      char **sock, int *default_deny, int *use_killswitch)
 {
     switch (c) {
     case 'h':
@@ -254,9 +240,7 @@ void read_serve_param( int c, char **ip_addr, char **ip_port, char **file, char
 void read_listen_param(int c,
 		       char **ip_addr,
 		       char **ip_port,
-		char **file,
+		       char **file, char **sock, int *default_deny)
-		char **sock,
-		int *default_deny )
 {
     switch (c) {
     case 'h':
@@ -289,7 +273,9 @@ void read_listen_param( int c,
     }
 }
 
-void read_readwrite_param( int c, char **ip_addr, char **ip_port, char **bind_addr, char **from, char **size, char *err_text )
+void read_readwrite_param(int c, char **ip_addr, char **ip_port,
+			  char **bind_addr, char **from, char **size,
+			  char *err_text)
 {
     switch (c) {
     case 'h':
@@ -348,11 +334,7 @@ void read_acl_param( int c, char **sock )
     read_sock_param(c, sock, acl_help_text);
 }
 
-void read_mirror_speed_param(
+void read_mirror_speed_param(int c, char **sock, char **max_speed)
-		int c,
-		char **sock,
-		char **max_speed
-)
 {
     switch (c) {
     case 'h':
@@ -377,13 +359,10 @@ void read_mirror_speed_param(
     }
 }
 
-void read_mirror_param(
+void read_mirror_param(int c,
-		int c,
 		       char **sock,
 		       char **ip_addr,
-		char **ip_port,
+		       char **ip_port, int *unlink, char **bind_addr)
-		int  *unlink,
-		char **bind_addr )
 {
     switch (c) {
     case 'h':
@@ -459,10 +438,14 @@ int mode_serve( int argc, char *argv[] )
     struct flexnbd *flexnbd;
 
     while (1) {
-		c = getopt_long(argc, argv, serve_short_options, serve_options, NULL);
+	c = getopt_long(argc, argv, serve_short_options, serve_options,
-		if ( c == -1 ) { break; }
+			NULL);
+	if (c == -1) {
+	    break;
+	}
 
-		read_serve_param( c, &ip_addr, &ip_port, &file, &sock, &default_deny, &use_killswitch );
+	read_serve_param(c, &ip_addr, &ip_port, &file, &sock,
+			 &default_deny, &use_killswitch);
     }
 
     if (NULL == ip_addr || NULL == ip_port) {
@@ -473,9 +456,14 @@ int mode_serve( int argc, char *argv[] )
 	err = 1;
 	fprintf(stderr, "--file is required\n");
     }
-	if ( err ) { exit_err( serve_help_text ); }
+    if (err) {
+	exit_err(serve_help_text);
+    }
 
-	flexnbd = flexnbd_create_serving( ip_addr, ip_port, file, sock, default_deny, argc - optind, argv + optind, MAX_NBD_CLIENTS, use_killswitch );
+    flexnbd =
+	flexnbd_create_serving(ip_addr, ip_port, file, sock, default_deny,
+			       argc - optind, argv + optind,
+			       MAX_NBD_CLIENTS, use_killswitch);
     info("Serving file %s", file);
     success = flexnbd_serve(flexnbd);
     flexnbd_destroy(flexnbd);
@@ -499,8 +487,11 @@ int mode_listen( int argc, char *argv[] )
     struct flexnbd *flexnbd;
 
     while (1) {
-		c = getopt_long(argc, argv, listen_short_options, listen_options, NULL);
+	c = getopt_long(argc, argv, listen_short_options, listen_options,
-		if ( c == -1 ) { break; }
+			NULL);
+	if (c == -1) {
+	    break;
+	}
 
 	read_listen_param(c, &ip_addr, &ip_port,
 			  &file, &sock, &default_deny);
@@ -514,16 +505,16 @@ int mode_listen( int argc, char *argv[] )
 	err = 1;
 	fprintf(stderr, "--file is required\n");
     }
-	if ( err ) { exit_err( listen_help_text ); }
+    if (err) {
+	exit_err(listen_help_text);
+    }
 
-	flexnbd = flexnbd_create_listening(
+    flexnbd = flexnbd_create_listening(ip_addr,
-		ip_addr,
 				       ip_port,
 				       file,
 				       sock,
 				       default_deny,
-		argc - optind,
+				       argc - optind, argv + optind);
-		argv + optind);
     success = flexnbd_serve(flexnbd);
     flexnbd_destroy(flexnbd);
 
@@ -545,29 +536,25 @@ int mode_listen( int argc, char *argv[] )
  *		 char *s_length,
  *		 char *s_filename )
  */
-void params_readwrite(
+void params_readwrite(int write_not_read,
-	int write_not_read,
 		      struct mode_readwrite_params *out,
 		      char *s_ip_address,
 		      char *s_port,
 		      char *s_bind_address,
-	char* s_from,
+		      char *s_from, char *s_length_or_filename)
-	char* s_length_or_filename
-)
 {
     FATAL_IF_NULL(s_ip_address, "No IP address supplied");
     FATAL_IF_NULL(s_port, "No port number supplied");
     FATAL_IF_NULL(s_from, "No from supplied");
     FATAL_IF_NULL(s_length_or_filename, "No length supplied");
 
-	FATAL_IF_ZERO(
+    FATAL_IF_ZERO(parse_ip_to_sockaddr
-		parse_ip_to_sockaddr(&out->connect_to.generic, s_ip_address),
+		  (&out->connect_to.generic, s_ip_address),
-		"Couldn't parse connection address '%s'",
+		  "Couldn't parse connection address '%s'", s_ip_address);
-		s_ip_address
-	);
 
     if (s_bind_address != NULL &&
-			parse_ip_to_sockaddr(&out->connect_from.generic, s_bind_address) == 0) {
+	parse_ip_to_sockaddr(&out->connect_from.generic,
+			     s_bind_address) == 0) {
 	fatal("Couldn't parse bind address '%s'", s_bind_address);
     }
 
@@ -575,30 +562,27 @@ void params_readwrite(
 
     long signed_from = atol(s_from);
     FATAL_IF_NEGATIVE(signed_from,
-			"Can't read from a negative offset %d.", signed_from);
+		      "Can't read from a negative offset %d.",
+		      signed_from);
     out->from = signed_from;
 
     if (write_not_read) {
 	if (s_length_or_filename[0] - 48 < 10) {
 	    out->len = atol(s_length_or_filename);
 	    out->data_fd = 0;
-		}
+	} else {
-		else {
+	    out->data_fd = open(s_length_or_filename, O_RDONLY);
-			out->data_fd = open(
-			  s_length_or_filename, O_RDONLY);
 	    FATAL_IF_NEGATIVE(out->data_fd,
 			      "Couldn't open %s", s_length_or_filename);
 	    off64_t signed_len = lseek64(out->data_fd, 0, SEEK_END);
 	    FATAL_IF_NEGATIVE(signed_len,
-			  "Couldn't find length of %s", s_length_or_filename);
+			      "Couldn't find length of %s",
+			      s_length_or_filename);
 	    out->len = signed_len;
-			FATAL_IF_NEGATIVE(
+	    FATAL_IF_NEGATIVE(lseek64(out->data_fd, 0, SEEK_SET),
-				lseek64(out->data_fd, 0, SEEK_SET),
+			      "Couldn't rewind %s", s_length_or_filename);
-				"Couldn't rewind %s", s_length_or_filename
-			);
 	}
-	}
+    } else {
-	else {
 	out->len = atol(s_length_or_filename);
 	out->data_fd = 1;
     }
@@ -618,11 +602,15 @@ int mode_read( int argc, char *argv[] )
     struct mode_readwrite_params readwrite;
 
     while (1) {
-		c = getopt_long(argc, argv, read_short_options, read_options, NULL);
+	c = getopt_long(argc, argv, read_short_options, read_options,
+			NULL);
 
-		if ( c == -1 ) { break; }
+	if (c == -1) {
+	    break;
+	}
 
-		read_readwrite_param( c, &ip_addr, &ip_port, &bind_addr, &from, &size, read_help_text );
+	read_readwrite_param(c, &ip_addr, &ip_port, &bind_addr, &from,
+			     &size, read_help_text);
     }
 
     if (NULL == ip_addr || NULL == ip_port) {
@@ -633,10 +621,13 @@ int mode_read( int argc, char *argv[] )
 	err = 1;
 	fprintf(stderr, "both --from and --size are required.\n");
     }
-	if ( err ) { exit_err( read_help_text ); }
+    if (err) {
+	exit_err(read_help_text);
+    }
 
     memset(&readwrite, 0, sizeof(readwrite));
-	params_readwrite( 0, &readwrite, ip_addr, ip_port, bind_addr, from, size );
+    params_readwrite(0, &readwrite, ip_addr, ip_port, bind_addr, from,
+		     size);
     do_read(&readwrite);
     return 0;
 }
@@ -654,10 +645,14 @@ int mode_write( int argc, char *argv[] )
     struct mode_readwrite_params readwrite;
 
     while (1) {
-		c = getopt_long(argc, argv, write_short_options, write_options, NULL);
+	c = getopt_long(argc, argv, write_short_options, write_options,
-		if ( c == -1 ) { break; }
+			NULL);
+	if (c == -1) {
+	    break;
+	}
 
-		read_readwrite_param( c, &ip_addr, &ip_port, &bind_addr, &from, &size, write_help_text );
+	read_readwrite_param(c, &ip_addr, &ip_port, &bind_addr, &from,
+			     &size, write_help_text);
     }
 
     if (NULL == ip_addr || NULL == ip_port) {
@@ -668,10 +663,13 @@ int mode_write( int argc, char *argv[] )
 	err = 1;
 	fprintf(stderr, "both --from and --size are required.\n");
     }
-	if ( err ) { exit_err( write_help_text ); }
+    if (err) {
+	exit_err(write_help_text);
+    }
 
     memset(&readwrite, 0, sizeof(readwrite));
-	params_readwrite( 1, &readwrite, ip_addr, ip_port, bind_addr, from, size );
+    params_readwrite(1, &readwrite, ip_addr, ip_port, bind_addr, from,
+		     size);
     do_write(&readwrite);
     return 0;
 }
@@ -683,7 +681,9 @@ int mode_acl( int argc, char *argv[] )
 
     while (1) {
 	c = getopt_long(argc, argv, acl_short_options, acl_options, NULL);
-		if ( c == -1 ) { break; }
+	if (c == -1) {
+	    break;
+	}
 	read_acl_param(c, &sock);
     }
 
@@ -708,8 +708,11 @@ int mode_mirror_speed( int argc, char *argv[] )
     char *speed = NULL;
 
     while (1) {
-		c = getopt_long( argc, argv, mirror_speed_short_options, mirror_speed_options, NULL );
+	c = getopt_long(argc, argv, mirror_speed_short_options,
-		if ( -1 == c ) { break; }
+			mirror_speed_options, NULL);
+	if (-1 == c) {
+	    break;
+	}
 	read_mirror_speed_param(c, &sock, &speed);
     }
 
@@ -739,14 +742,15 @@ int mode_mirror( int argc, char *argv[] )
     remote_argv[2] = "exit";
 
     while (1) {
-		c = getopt_long( argc, argv, mirror_short_options, mirror_options, NULL);
+	c = getopt_long(argc, argv, mirror_short_options, mirror_options,
-		if ( -1 == c ) { break; }
+			NULL);
+	if (-1 == c) {
+	    break;
+	}
 	read_mirror_param(c,
 			  &sock,
 			  &remote_argv[0],
-				&remote_argv[1],
+			  &remote_argv[1], &unlink, &remote_argv[3]);
-				&unlink,
-				&remote_argv[3] );
     }
 
     if (NULL == sock) {
@@ -757,13 +761,16 @@ int mode_mirror( int argc, char *argv[] )
 	fprintf(stderr, "both --addr and --port are required.\n");
 	err = 1;
     }
-	if ( err ) { exit_err( mirror_help_text ); }
+    if (err) {
-	if ( unlink ) { remote_argv[2] = "unlink"; }
+	exit_err(mirror_help_text);
+    }
+    if (unlink) {
+	remote_argv[2] = "unlink";
+    }
 
     if (remote_argv[3] == NULL) {
 	do_remote_command("mirror", sock, 3, remote_argv);
-	}
+    } else {
-	else {
 	do_remote_command("mirror", sock, 4, remote_argv);
     }
 
@@ -777,8 +784,11 @@ int mode_break( int argc, char *argv[] )
     char *sock = NULL;
 
     while (1) {
-		c = getopt_long( argc, argv, break_short_options, break_options, NULL );
+	c = getopt_long(argc, argv, break_short_options, break_options,
-		if ( -1 == c ) { break; }
+			NULL);
+	if (-1 == c) {
+	    break;
+	}
 	read_break_param(c, &sock);
     }
 
@@ -798,8 +808,11 @@ int mode_status( int argc, char *argv[] )
     char *sock = NULL;
 
     while (1) {
-		c = getopt_long( argc, argv, status_short_options, status_options, NULL );
+	c = getopt_long(argc, argv, status_short_options, status_options,
-		if ( -1 == c ) { break; }
+			NULL);
+	if (-1 == c) {
+	    break;
+	}
 	read_status_param(c, &sock);
     }
 
@@ -836,7 +849,9 @@ int mode_help( int argc, char *argv[] )
 	    help_text = mirror_help_text;
 	} else if (IS_CMD(CMD_STATUS, cmd)) {
 	    help_text = status_help_text;
-		} else { exit_err( help_help_text ); }
+	} else {
+	    exit_err(help_help_text);
+	}
     }
 
     fprintf(stdout, "%s\n", help_text);
@@ -848,37 +863,27 @@ void mode(char* mode, int argc, char **argv)
 {
     if (IS_CMD(CMD_SERVE, mode)) {
 	exit(mode_serve(argc, argv));
-	}
+    } else if (IS_CMD(CMD_LISTEN, mode)) {
-	else if ( IS_CMD( CMD_LISTEN, mode ) ) {
 	exit(mode_listen(argc, argv));
-	}
+    } else if (IS_CMD(CMD_READ, mode)) {
-	else if ( IS_CMD( CMD_READ, mode ) ) {
 	mode_read(argc, argv);
-	}
+    } else if (IS_CMD(CMD_WRITE, mode)) {
-	else if ( IS_CMD( CMD_WRITE, mode ) ) {
 	mode_write(argc, argv);
-	}
+    } else if (IS_CMD(CMD_ACL, mode)) {
-	else if ( IS_CMD( CMD_ACL, mode ) ) {
 	mode_acl(argc, argv);
     } else if (IS_CMD(CMD_MIRROR_SPEED, mode)) {
 	mode_mirror_speed(argc, argv);
-	}
+    } else if (IS_CMD(CMD_MIRROR, mode)) {
-	else if ( IS_CMD( CMD_MIRROR, mode ) ) {
 	mode_mirror(argc, argv);
-	}
+    } else if (IS_CMD(CMD_BREAK, mode)) {
-       	else if ( IS_CMD( CMD_BREAK, mode ) ) {
 	mode_break(argc, argv);
-	}
+    } else if (IS_CMD(CMD_STATUS, mode)) {
-       	else if ( IS_CMD( CMD_STATUS, mode ) ) {
 	mode_status(argc, argv);
-	}
+    } else if (IS_CMD(CMD_HELP, mode)) {
-	else if ( IS_CMD( CMD_HELP, mode ) ) {
 	mode_help(argc - 1, argv + 1);
-	}
+    } else {
-	else {
 	mode_help(argc - 1, argv + 1);
 	exit(1);
     }
     exit(0);
 }
-

src/server/serve.c

@@ -21,8 +21,7 @@
 #include <sys/socket.h>
 #include <netinet/tcp.h>
 
-struct server * server_create (
+struct server *server_create(struct flexnbd *flexnbd,
-	struct flexnbd * flexnbd,
 			     char *s_ip_address,
 			     char *s_port,
 			     char *s_file,
@@ -30,8 +29,7 @@ struct server * server_create (
 			     int acl_entries,
 			     char **s_acl_entries,
 			     int max_nbd_clients,
-	int use_killswitch,
+			     int use_killswitch, int success)
-	int success)
 {
     NULLCHECK(flexnbd);
     struct server *out;
@@ -43,19 +41,18 @@ struct server * server_create (
 
     server_allow_new_clients(out);
 
-	out->nbd_client = xmalloc( max_nbd_clients * sizeof( struct client_tbl_entry ) );
+    out->nbd_client =
+	xmalloc(max_nbd_clients * sizeof(struct client_tbl_entry));
     out->tcp_backlog = 10;	/* does this need to be settable? */
 
     FATAL_IF_NULL(s_ip_address, "No IP address supplied");
     FATAL_IF_NULL(s_port, "No port number supplied");
     FATAL_IF_NULL(s_file, "No filename supplied");
     NULLCHECK(s_ip_address);
-	FATAL_IF_ZERO(
+    FATAL_IF_ZERO(parse_ip_to_sockaddr
-		parse_ip_to_sockaddr(&out->bind_to.generic, s_ip_address),
+		  (&out->bind_to.generic, s_ip_address),
 		  "Couldn't parse server address '%s' (use 0 if "
-		"you want to bind to all IPs)",
+		  "you want to bind to all IPs)", s_ip_address);
-		s_ip_address
-	);
 
 
     out->acl = acl_create(acl_entries, s_acl_entries, default_deny);
@@ -110,8 +107,7 @@ void server_unlink( struct server * serve )
 
     FATAL_IF_NEGATIVE(unlink(serve->filename),
 		      "Failed to unlink %s: %s",
-			serve->filename,
+		      serve->filename, strerror(errno));
-			strerror( errno ) );
 
 }
 
@@ -155,7 +151,8 @@ void server_unlock_start_mirror( struct server *serve )
 {
     debug("Mirror start unlocking");
 
-	SERVER_UNLOCK( serve, l_start_mirror, "Problem with start mirror unlock" );
+    SERVER_UNLOCK(serve, l_start_mirror,
+		  "Problem with start mirror unlock");
 }
 
 int server_start_mirror_locked(struct server *serve)
@@ -186,8 +183,9 @@ void serve_open_server_socket(struct server* params)
 {
     NULLCHECK(params);
 
-	params->server_fd = socket(params->bind_to.generic.sa_family == AF_INET ?
+    params->server_fd =
-	  PF_INET : PF_INET6, SOCK_STREAM, 0);
+	socket(params->bind_to.generic.sa_family ==
+	       AF_INET ? PF_INET : PF_INET6, SOCK_STREAM, 0);
 
     FATAL_IF_NEGATIVE(params->server_fd, "Couldn't create server socket");
 
@@ -199,35 +197,32 @@ void serve_open_server_socket(struct server* params)
      * problem.  It's also indicative of something odd going on, so
      * we barf.
      */
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(sock_set_reuseaddr(params->server_fd, 1),
-		sock_set_reuseaddr( params->server_fd, 1 ), "Couldn't set SO_REUSEADDR"
+		      "Couldn't set SO_REUSEADDR");
-	);
 
     /* TCP_NODELAY makes everything not be slow.  If we can't set
      * this, again, there's something odd going on which we don't
      * understand.
      */
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(sock_set_tcp_nodelay(params->server_fd, 1),
-		sock_set_tcp_nodelay( params->server_fd, 1 ), "Couldn't set TCP_NODELAY"
+		      "Couldn't set TCP_NODELAY");
-	);
 
     /* If we can't bind, presumably that's because someone else is
      * squatting on our ip/port combo, or the ip isn't yet
      * configured. Ideally we want to retry this. */
-	FATAL_UNLESS_ZERO(
+    FATAL_UNLESS_ZERO(sock_try_bind
-		sock_try_bind( params->server_fd, &params->bind_to.generic ),
+		      (params->server_fd, &params->bind_to.generic),
 		      SHOW_ERRNO("Failed to bind() socket")
 	);
 
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(listen(params->server_fd, params->tcp_backlog),
-		listen(params->server_fd, params->tcp_backlog),
+		      "Couldn't listen on server socket");
-		"Couldn't listen on server socket"
-	);
 }
 
 
 
-int tryjoin_client_thread( struct client_tbl_entry *entry, int (*joinfunc)(pthread_t, void **) )
+int tryjoin_client_thread(struct client_tbl_entry *entry,
+			  int (*joinfunc) (pthread_t, void **))
 {
 
     NULLCHECK(entry);
@@ -239,25 +234,25 @@ int tryjoin_client_thread( struct client_tbl_entry *entry, int (*joinfunc)(pthre
     if (entry->thread != 0) {
 	char s_client_address[128];
 
-		sockaddr_address_string( &entry->address.generic, &s_client_address[0], 128 );
+	sockaddr_address_string(&entry->address.generic,
+				&s_client_address[0], 128);
 
-		debug( "%s(%p,...)", joinfunc == pthread_join ? "joining" : "tryjoining", entry->thread );
+	debug("%s(%p,...)",
+	      joinfunc == pthread_join ? "joining" : "tryjoining",
+	      entry->thread);
 	int join_errno = joinfunc(entry->thread, &status);
 
 	/* join_errno can legitimately be ESRCH if the thread is
 	 * already dead, but the client still needs tidying up. */
 	if (join_errno != 0 && !entry->client->stopped) {
-			debug( "join_errno was %s, stopped was %d", strerror( join_errno ), entry->client->stopped );
+	    debug("join_errno was %s, stopped was %d",
+		  strerror(join_errno), entry->client->stopped);
 	    FATAL_UNLESS(join_errno == EBUSY,
 			 "Problem with joining thread %p: %s",
-					entry->thread,
+			 entry->thread, strerror(join_errno));
-					strerror(join_errno) );
+	} else if (join_errno == 0) {
-		}
-		else if ( join_errno == 0 ) {
 	    debug("nbd thread %016x exited (%s) with status %ld",
-					entry->thread,
+		  entry->thread, s_client_address, (uintptr_t) status);
-					s_client_address,
-					(uintptr_t)status);
 	    client_destroy(entry->client);
 	    entry->client = NULL;
 	    entry->thread = 0;
@@ -287,7 +282,8 @@ int cleanup_client_thread( struct client_tbl_entry * entry )
     return tryjoin_client_thread(entry, pthread_tryjoin_np);
 }
 
-void cleanup_client_threads( struct client_tbl_entry * entries, size_t entries_len )
+void cleanup_client_threads(struct client_tbl_entry *entries,
+			    size_t entries_len)
 {
     size_t i;
     for (i = 0; i < entries_len; i++) {
@@ -333,6 +329,8 @@ int server_count_clients( struct server *params )
     NULLCHECK(params);
     int i, count = 0;
 
+    cleanup_client_threads(params->nbd_client, params->max_nbd_clients);
+
     for (i = 0; i < params->max_nbd_clients; i++) {
 	if (params->nbd_client[i].thread != 0) {
 	    count++;
@@ -347,7 +345,8 @@ int server_count_clients( struct server *params )
  * to the current acl.  If params->acl is NULL, the result will be 1,
  * otherwise it will be the result of acl_includes().
  */
-int server_acl_accepts( struct server *params, union mysockaddr * client_address )
+int server_acl_accepts(struct server *params,
+		       union mysockaddr *client_address)
 {
     NULLCHECK(params);
     NULLCHECK(client_address);
@@ -366,8 +365,7 @@ int server_acl_accepts( struct server *params, union mysockaddr * client_address
 }
 
 
-int server_should_accept_client(
+int server_should_accept_client(struct server *params,
-		struct server * params,
 				union mysockaddr *client_address,
 				char *s_client_address,
 				size_t s_client_address_len)
@@ -376,9 +374,9 @@ int server_should_accept_client(
     NULLCHECK(client_address);
     NULLCHECK(s_client_address);
 
-	const char* result = sockaddr_address_string(
+    const char *result =
-		&client_address->generic, s_client_address, s_client_address_len
+	sockaddr_address_string(&client_address->generic, s_client_address,
-	);
+				s_client_address_len);
 
     if (NULL == result) {
 	warn("Rejecting client %s: Bad client_address", s_client_address);
@@ -386,7 +384,8 @@ int server_should_accept_client(
     }
 
     if (!server_acl_accepts(params, client_address)) {
-		warn( "Rejecting client %s: Access control error", s_client_address );
+	warn("Rejecting client %s: Access control error",
+	     s_client_address);
 	debug("We %s have an acl, and default_deny is %s",
 	      (params->acl ? "do" : "do not"),
 	      (params->acl->default_deny ? "true" : "false"));
@@ -398,11 +397,11 @@ int server_should_accept_client(
 
 
 
-int spawn_client_thread(
+int spawn_client_thread(struct client *client_params,
-	struct client * client_params,
 			pthread_t * out_thread)
 {
-	int result = pthread_create(out_thread, NULL, client_serve, client_params);
+    int result =
+	pthread_create(out_thread, NULL, client_serve, client_params);
 
     return result;
 }
@@ -412,10 +411,8 @@ int spawn_client_thread(
   * to handle it.  Rejects the connection if there is an ACL, and the far end's
   * address doesn't match, or if there are too many clients already connected.
   */
-void accept_nbd_client(
+void accept_nbd_client(struct server *params,
-		struct server* params,
+		       int client_fd, union mysockaddr *client_address)
-		int client_fd,
-		union mysockaddr* client_address)
 {
     NULLCHECK(params);
     NULLCHECK(client_address);
@@ -424,11 +421,15 @@ void accept_nbd_client(
     int slot;
     char s_client_address[64] = { 0 };
 
-	FATAL_IF_NEGATIVE( sock_set_keepalive_params( client_fd, CLIENT_KEEPALIVE_TIME, CLIENT_KEEPALIVE_INTVL, CLIENT_KEEPALIVE_PROBES),
+    FATAL_IF_NEGATIVE(sock_set_keepalive_params
-		"Error setting keepalive parameters on client socket fd %d", client_fd );
+		      (client_fd, CLIENT_KEEPALIVE_TIME,
+		       CLIENT_KEEPALIVE_INTVL, CLIENT_KEEPALIVE_PROBES),
+		      "Error setting keepalive parameters on client socket fd %d",
+		      client_fd);
 
 
-	if ( !server_should_accept_client( params, client_address, s_client_address, 64 ) ) {
+    if (!server_should_accept_client
+	(params, client_address, s_client_address, 64)) {
 	FATAL_IF_NEGATIVE(close(client_fd),
 			  "Error closing client socket fd %d", client_fd);
 	debug("Closed client socket fd %d", client_fd);
@@ -462,7 +463,8 @@ void accept_nbd_client(
 	return;
     }
 
-	debug("nbd thread %p started (%s)", params->nbd_client[slot].thread, s_client_address);
+    debug("nbd thread %p started (%s)", params->nbd_client[slot].thread,
+	  s_client_address);
 }
 
 
@@ -483,8 +485,12 @@ void server_audit_clients( struct server * serve)
      */
     for (i = 0; i < serve->max_nbd_clients; i++) {
 	entry = &serve->nbd_client[i];
-		if ( 0 == entry->thread ) { continue; }
+	if (0 == entry->thread) {
-		if ( server_acl_accepts( serve, &entry->address ) ) { continue; }
+	    continue;
+	}
+	if (server_acl_accepts(serve, &entry->address)) {
+	    continue;
+	}
 	client_signal_stop(entry->client);
     }
 }
@@ -544,7 +550,9 @@ void server_replace_acl( struct server *serve, struct acl * new_acl )
 	struct acl *old_acl = serve->acl;
 	serve->acl = new_acl;
 	/* We should always have an old_acl, but just in case... */
-		if ( old_acl ) { acl_destroy( old_acl ); }
+	if (old_acl) {
+	    acl_destroy(old_acl);
+	}
     }
     server_unlock_acl(serve);
 
@@ -612,12 +620,13 @@ int server_accept( struct server * params )
 
     FD_ZERO(&fds);
     FD_SET(params->server_fd, &fds);
-	if( 0 <  signal_fd ) { FD_SET(signal_fd, &fds); }
+    if (0 < signal_fd) {
+	FD_SET(signal_fd, &fds);
+    }
     self_pipe_fd_set(params->close_signal, &fds);
     self_pipe_fd_set(params->acl_updated_signal, &fds);
 
-	FATAL_IF_NEGATIVE(
+    FATAL_IF_NEGATIVE(sock_try_select(FD_SETSIZE, &fds, NULL, NULL, NULL),
-		sock_try_select(FD_SETSIZE, &fds, NULL, NULL, NULL),
 		      SHOW_ERRNO("select() failed")
 	);
 
@@ -630,7 +639,8 @@ int server_accept( struct server * params )
     if (0 < signal_fd && FD_ISSET(signal_fd, &fds)) {
 	debug("Stop signal received.");
 	server_close_clients(params);
-		params->success = params->success && serve_shutdown_is_graceful( params );
+	params->success = params->success
+	    && serve_shutdown_is_graceful(params);
 	should_continue = 0;
     }
 
@@ -641,7 +651,8 @@ int server_accept( struct server * params )
     }
 
     if (FD_ISSET(params->server_fd, &fds)) {
-		int client_fd = accept( params->server_fd, &client_address.generic, &socklen );
+	int client_fd =
+	    accept(params->server_fd, &client_address.generic, &socklen);
 
 	if (params->allow_new_clients) {
 	    debug("Accepted nbd client socket fd %d", client_fd);
@@ -676,8 +687,7 @@ void* build_allocation_map_thread(void* serve_uncast)
 
     if (build_allocation_map(serve->allocation_map, fd)) {
 	serve->allocation_map_built = 1;
-	}
+    } else {
-	else {
 	/* We can operate without it, but we can't free it without a race.
 	 * All that happens if we leave it is that it gradually builds up an
 	 * *incomplete* record of writes. Nobody will use it, as
@@ -710,9 +720,17 @@ void serve_init_allocation_map(struct server* params)
 
     FATAL_IF_NEGATIVE(fd, "Couldn't open %s", params->filename);
     size = lseek64(fd, 0, SEEK_END);
+
+    /* If discs are not in multiples of 512, then odd things happen,
+     * resulting in reads/writes past the ends of files.
+     */
+    if (size != (size & ~0x1ff)) {
+	warn("file does not fit into 512-byte sectors; the end of the file will be ignored.");
+	size &= ~0x1ff;
+    }
+
     params->size = size;
-	FATAL_IF_NEGATIVE( size, "Couldn't find size of %s",
+    FATAL_IF_NEGATIVE(size, "Couldn't find size of %s", params->filename);
-			   params->filename );
 
     params->allocation_map =
 	bitset_alloc(params->size, block_allocation_resolution);
@@ -738,7 +756,8 @@ void server_allow_new_clients( struct server * serve )
     return;
 }
 
-void server_join_clients( struct server * serve ) {
+void server_join_clients(struct server *serve)
+{
     int i;
     void *status;
 
@@ -751,7 +770,8 @@ void server_join_clients( struct server * serve ) {
 	    if (0 == err) {
 		serve->nbd_client[i].thread = 0;
 	    } else {
-				warn( "Error %s (%i) joining thread %p", strerror( err ), err, thread_id );
+		warn("Error %s (%i) joining thread %p", strerror(err), err,
+		     thread_id);
 	    }
 	}
     }
@@ -792,8 +812,6 @@ void server_control_arrived( struct server *serve )
 }
 
 
-void flexnbd_stop_control( struct flexnbd * flexnbd );
-
 /** Closes sockets, frees memory and waits for all client threads to finish */
 void serve_cleanup(struct server *params,
 		   int fatal __attribute__ ((unused)))
@@ -803,7 +821,21 @@ void serve_cleanup(struct server* params,
 
     info("cleaning up");
     
-	if (params->server_fd){ close(params->server_fd); }
+    /* Close the control socket, and wait for it to close before proceeding.
+     * If we do not wait, we risk a race condition with the tail supervisor
+     * sending a status command, and deadlocking the mirroring.  */
+    if (params->flexnbd && params->flexnbd->control) {
+	debug("closing control socket");
+        control_signal_close(params->flexnbd->control);
+
+	debug("waiting for control socket to close");
+	control_wait_for_close(params->flexnbd->control);
+    }
+    
+    if (params->server_fd) {
+	debug("closing server_fd");
+	close(params->server_fd);
+    }
 
     /* need to stop background build if we're killed very early on */
     pthread_cancel(params->allocation_map_builder_thread);
@@ -812,14 +844,18 @@ void serve_cleanup(struct server* params,
     int need_mirror_lock;
     need_mirror_lock = !server_start_mirror_locked(params);
 
-	if ( need_mirror_lock ) { server_lock_start_mirror( params ); }
+    if (need_mirror_lock) {
+	server_lock_start_mirror(params);
+    }
     {
 	if (server_is_mirroring(params)) {
 	    server_abandon_mirror(params);
 	}
 	server_prevent_mirror_start(params);
     }
-	if ( need_mirror_lock ) { server_unlock_start_mirror( params ); }
+    if (need_mirror_lock) {
+	server_unlock_start_mirror(params);
+    }
 
     server_join_clients(params);
 
@@ -835,15 +871,6 @@ void serve_cleanup(struct server* params,
 	server_unlock_acl(params);
     }
 
-	/* if( params->flexnbd ) { */
-	/* 	if ( params->flexnbd->control ) { */
-	/* 		flexnbd_stop_control( params->flexnbd ); */
-	/* 	} */
-	/* 	flexnbd_destroy( params->flexnbd ); */
-	/* } */
-
-	/* server_destroy( params ); */
-
     debug("Cleanup done");
 }
 
@@ -864,8 +891,11 @@ uint64_t server_mirror_bytes_remaining( struct server * serve )
 {
     if (server_is_mirroring(serve)) {
 	uint64_t bytes_to_xfer =
-			bitset_stream_queued_bytes( serve->allocation_map, BITSET_STREAM_SET ) +
+	    bitset_stream_queued_bytes(serve->allocation_map,
-			( serve->size - serve->mirror->offset );
+				       BITSET_STREAM_SET) + (serve->size -
+							     serve->
+							     mirror->
+							     offset);
 
 	return bytes_to_xfer;
     }
@@ -911,10 +941,8 @@ void server_abandon_mirror( struct server * serve )
 	 * We can set abandon_signal after mirror_super has checked it, but
 	 * before the reset. However, mirror_reset doesn't clear abandon_signal
 	 * so it'll just terminate early on the next pass. */
-		ERROR_UNLESS(
+	ERROR_UNLESS(self_pipe_signal(serve->mirror->abandon_signal),
-			self_pipe_signal( serve->mirror->abandon_signal ),
+		     "Failed to signal abandon to mirror");
-			"Failed to signal abandon to mirror"
-		);
 
 	pthread_t tid = serve->mirror_super->thread;
 	pthread_join(tid, NULL);
@@ -948,7 +976,9 @@ int do_serve( struct server* params, struct self_pipe * open_signal )
 
     /* Only signal that we are open for business once the server
        socket is open */
-	if ( NULL != open_signal ) { self_pipe_signal( open_signal ); }
+    if (NULL != open_signal) {
+	self_pipe_signal(open_signal);
+    }
 
     serve_init_allocation_map(params);
     serve_accept_loop(params);

src/server/serve.h

@@ -103,8 +103,7 @@ struct server {
     int success;
 };
 
-struct server * server_create(
+struct server *server_create(struct flexnbd *flexnbd,
-		struct flexnbd * flexnbd,
 			     char *s_ip_address,
 			     char *s_port,
 			     char *s_file,
@@ -112,8 +111,7 @@ struct server * server_create(
 			     int acl_entries,
 			     char **s_acl_entries,
 			     int max_nbd_clients,
-		int use_killswitch,
+			     int use_killswitch, int success);
-		int success );
 void server_destroy(struct server *);
 int server_is_closed(struct server *serve);
 void serve_signal_close(struct server *serve);
@@ -167,4 +165,3 @@ struct mode_readwrite_params {
 
 
 #endif
-

src/server/status.c

@@ -21,17 +21,20 @@ struct status * status_create( struct server * serve )
     if (status->is_mirroring) {
 	status->migration_duration = monotonic_time_ms();
 
-		if ( ( serve->mirror->migration_started ) < status->migration_duration ) {
+	if ((serve->mirror->migration_started) <
+	    status->migration_duration) {
 	    status->migration_duration -= serve->mirror->migration_started;
 	} else {
 	    status->migration_duration = 0;
 	}
 	status->migration_duration /= 1000;
 	status->migration_speed = server_mirror_bps(serve);
-		status->migration_speed_limit = serve->mirror->max_bytes_per_second;
+	status->migration_speed_limit =
+	    serve->mirror->max_bytes_per_second;
 
 	status->migration_seconds_left = server_mirror_eta(serve);
-		status->migration_bytes_left = server_mirror_bytes_remaining( serve );
+	status->migration_bytes_left =
+	    server_mirror_bytes_remaining(serve);
     }
 
     server_unlock_start_mirror(serve);
@@ -77,4 +80,3 @@ void status_destroy( struct status * status )
     NULLCHECK(status);
     free(status);
 }
-

src/server/status.h

@@ -101,4 +101,3 @@ void status_destroy( struct status * );
 
 
 #endif
-

tests/acceptance/environment.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 require 'flexnbd'
 require 'file_writer'
 
@@ -9,18 +7,23 @@ class Environment
 
   def initialize
     @blocksize = 1024
-    @filename1 = "/tmp/.flexnbd.test.#{$$}.#{Time.now.to_i}.1"
+    @filename1 = "/tmp/.flexnbd.test.#{$PROCESS_ID}.#{Time.now.to_i}.1"
-    @filename2 = "/tmp/.flexnbd.test.#{$$}.#{Time.now.to_i}.2"
+    @filename2 = "/tmp/.flexnbd.test.#{$PROCESS_ID}.#{Time.now.to_i}.2"
-    @ip = "127.0.0.1"
+    @ip = '127.0.0.1'
-    @available_ports = [*40000..41000] - listening_ports
+    @available_ports = [*40_000..41_000] - listening_ports
     @port1 = @available_ports.shift
     @port2 = @available_ports.shift
-    @nbd1 = FlexNBD::FlexNBD.new("../../build/flexnbd", @ip, @port1)
+    @nbd1 = FlexNBD::FlexNBD.new('../../build/flexnbd', @ip, @port1)
-    @nbd2 = FlexNBD::FlexNBD.new("../../build/flexnbd", @ip, @port2)
+    @nbd2 = FlexNBD::FlexNBD.new('../../build/flexnbd', @ip, @port2)
 
     @fake_pid = nil
   end
 
+  def blocksize=(b)
+    raise RuntimeError, "Unable to change blocksize after files have been opened" if @file1 or @file2
+    @blocksize = b
+  end
+
   def prefetch_proxy!
     @nbd1.prefetch_proxy = true
     @nbd2.prefetch_proxy = true
@@ -29,11 +32,11 @@ class Environment
   def proxy1(port = @port2)
     @nbd1.proxy(@ip, port)
   end
+
   def proxy2(port = @port1)
     @nbd2.proxy(@ip, port)
   end
 
-
   def serve1(*acl)
     @nbd1.serve(@filename1, *acl)
   end
@@ -42,7 +45,6 @@ class Environment
     @nbd2.serve(@filename2, *acl)
   end
 
-
   def listen1(*acl)
     @nbd1.listen(@filename1, *(acl.empty? ? @acl1 : acl))
   end
@@ -51,7 +53,6 @@ class Environment
     @nbd2.listen(@filename2, *acl)
   end
 
-
   def break1
     @nbd1.break
   end
@@ -64,7 +65,6 @@ class Environment
     @nbd2.acl(*acl)
   end
 
-
   def status1
     @nbd1.status.first
   end
@@ -73,8 +73,6 @@ class Environment
     @nbd2.status.first
   end
 
-
-
   def mirror12
     @nbd1.mirror(@nbd2.ip, @nbd2.port)
   end
@@ -87,7 +85,6 @@ class Environment
     @nbd1.mirror_unlink(@nbd2.ip, @nbd2.port, 2)
   end
 
-
   def write1(data)
     @nbd1.write(0, data)
   end
@@ -100,20 +97,17 @@ class Environment
     @file2 = FileWriter.new(@filename2, @blocksize).write(data)
   end
 
-
   def truncate1(size)
     system "truncate -s #{size} #{@filename1}"
   end
 
-
   def listening_ports
-    `netstat -ltn`.
+    `netstat -ltn`
-      split("\n").
+      .split("\n")
-      map { |x| x.split(/\s+/) }[2..-1].
+      .map { |x| x.split(/\s+/) }[2..-1]
-      map { |l| l[3].split(":")[-1].to_i }
+      .map { |l| l[3].split(':')[-1].to_i }
   end
 
-
   def cleanup
     if @fake_pid
       begin
@@ -122,41 +116,35 @@ class Environment
       end
     end
 
-
     @nbd1.can_die(0)
     @nbd1.kill
     @nbd2.kill
 
     [@filename1, @filename2].each do |f|
-      File.unlink(f) if File.exists?(f)
+      File.unlink(f) if File.exist?(f)
     end
   end
 
-
   def run_fake(name, addr, port, sock = nil)
-    fakedir = File.join( File.dirname( __FILE__ ), "fakes" )
+    fakedir = File.join(File.dirname(__FILE__), 'fakes')
-    fakeglob = File.join( fakedir, name ) + "*"
+    fakeglob = File.join(fakedir, name) + '*'
-    fake = Dir[fakeglob].sort.find { |fn|
+    fake = Dir[fakeglob].sort.find do |fn|
       File.executable?(fn)
-    }
+    end
 
     raise "no fake executable at #{fakeglob}" unless fake
-    raise "no addr" unless addr
+    raise 'no addr' unless addr
-    raise "no port" unless port
+    raise 'no port' unless port
 
     @fake_pid = fork do
-      exec [fake, addr, port, @nbd1.pid, sock].map{|x| x.to_s}.join(" ")
+      exec [fake, addr, port, @nbd1.pid, sock].map(&:to_s).join(' ')
     end
     sleep(0.5)
   end
 
-
   def fake_reports_success
     _, status = Process.waitpid2(@fake_pid)
     @fake_pid = nil
     status.success?
   end
-
-
 end # class Environment
-

tests/acceptance/fakes/dest/break_after_hello.rb

@@ -1,6 +1,4 @@
 #!/usr/bin/env ruby
-# encoding: utf-8
-
 # Open a server, accept a client, then cancel the migration by issuing
 # a break command.
 
@@ -13,23 +11,22 @@ client = server.accept
 
 ctrl = UNIXSocket.open(sock)
 
-Process.kill("STOP", src_pid.to_i)
+Process.kill('STOP', src_pid.to_i)
 ctrl.write("break\n")
 ctrl.close_write
 client.write_hello
-Process.kill("CONT", src_pid.to_i)
+Process.kill('CONT', src_pid.to_i)
 
-fail "Unexpected control response" unless
+raise 'Unexpected control response' unless
   ctrl.read =~ /0: mirror stopped/
 
 client2 = nil
 begin
-  client2 = server.accept( "Expected timeout" )
+  client2 = server.accept('Expected timeout')
-  fail "Unexpected reconnection"
+  raise 'Unexpected reconnection'
 rescue Timeout::Error
   # expected
 end
 client.close
 
 exit(0)
-

tests/acceptance/fakes/dest/close_after_entrust_reply.rb

@@ -1,6 +1,4 @@
 #!/usr/bin/env ruby
-# encoding: utf-8
-
 # Receive a mirror, and disconnect after sending the entrust reply but
 # before it can send the disconnect signal.
 #
@@ -15,7 +13,7 @@ server = FakeDest.new( addr, port )
 client = server.accept
 
 client.write_hello
-while (req = client.read_request; req[:type] == 1)
+while req = client.read_request; req[:type] == 1
                                  client.read_data(req[:len])
                                  client.write_reply(req[:handle])
 end
@@ -26,11 +24,10 @@ client.close
 system "kill -CONT #{src_pid}"
 
 sleep(0.25)
-client2 = server.accept( "Timed out waiting for a reconnection" )
+client2 = server.accept('Timed out waiting for a reconnection')
 
 client2.close
 server.close
 
-$stderr.puts "done"
+warn 'done'
 exit(0)
-

tests/acceptance/fakes/dest/close_after_hello.rb

@@ -11,11 +11,11 @@ include FlexNBD
 
 addr, port = *ARGV
 server = FakeDest.new(addr, port)
-client = server.accept( "Timed out waiting for a connection" )
+client = server.accept('Timed out waiting for a connection')
 client.write_hello
 client.close
 
-new_client = server.accept( "Timed out waiting for a reconnection" )
+new_client = server.accept('Timed out waiting for a reconnection')
 new_client.close
 
 server.close

tests/acceptance/fakes/dest/close_after_write.rb

@@ -12,12 +12,12 @@ include FlexNBD
 
 addr, port = *ARGV
 server = FakeDest.new(addr, port)
-client = server.accept( "Timed out waiting for a connection" )
+client = server.accept('Timed out waiting for a connection')
 client.write_hello
 client.read_request
 client.close
 
-new_client = server.accept( "Timed out waiting for a reconnection" )
+new_client = server.accept('Timed out waiting for a reconnection')
 new_client.close
 
 server.close

tests/acceptance/fakes/dest/close_after_writes.rb

@@ -1,6 +1,4 @@
 #!/usr/bin/env ruby
-# encoding: utf-8
-
 # Open a server, accept a client, then we expect a single write
 # followed by an entrust.  However, we disconnect after the write so
 # the entrust will fail.  We don't expect a reconnection: the sender
@@ -17,9 +15,9 @@ client.write_hello
 req = client.read_request
 data = client.read_data(req[:len])
 
-Process.kill("STOP", src_pid.to_i)
+Process.kill('STOP', src_pid.to_i)
 client.write_reply(req[:handle], 0)
 client.close
-Process.kill("CONT", src_pid.to_i)
+Process.kill('CONT', src_pid.to_i)
 
 exit(0)

tests/acceptance/fakes/dest/error_on_write.rb

@@ -1,6 +1,4 @@
 #!/usr/bin/env ruby
-# encoding: utf-8
-
 require 'flexnbd/fake_dest'
 include FlexNBD
 
@@ -12,8 +10,7 @@ client.write_hello
 handle = client.read_request[:handle]
 client.write_error(handle)
 
-
+client2 = server.accept('Timed out waiting for a reconnection')
-client2 = server.accept( "Timed out waiting for a reconnection" )
 
 client.close
 client2.close

tests/acceptance/fakes/dest/hang_after_connect.rb

@@ -26,8 +26,8 @@ client.close
 # Invert the sense of the timeout exception, since we *don't* want a
 # connection attempt
 begin
-  server.accept( "Expected timeout" )
+  server.accept('Expected timeout')
-  fail "Unexpected reconnection"
+  raise 'Unexpected reconnection'
 rescue Timeout::Error
   # expected
 end

tests/acceptance/fakes/dest/hang_after_write.rb

@@ -1,6 +1,4 @@
 #!/usr/bin/env ruby
-# encoding: utf-8
-
 # Open a socket, say hello, receive a write, then sleep for >
 # MS_REQUEST_LIMIT_SECS seconds. This should tell the source that the
 # write has gone MIA, and we expect a reconnect.
@@ -15,12 +13,12 @@ client1.write_hello
 client1.read_request
 
 t = Thread.start do
-  client2 = server.accept( "Timed out waiting for a reconnection",
+  client2 = server.accept('Timed out waiting for a reconnection',
                           FlexNBD::MS_REQUEST_LIMIT_SECS + 2)
   client2.close
 end
 
-sleep_time = if ENV.has_key?('FLEXNBD_MS_REQUEST_LIMIT_SECS')
+sleep_time = if ENV.key?('FLEXNBD_MS_REQUEST_LIMIT_SECS')
                ENV['FLEXNBD_MS_REQUEST_LIMIT_SECS'].to_f
              else
                FlexNBD::MS_REQUEST_LIMIT_SECS

tests/acceptance/fakes/dest/hello_wrong_magic.rb

@@ -13,15 +13,15 @@ client1 = server.accept
 # We don't expect a reconnection attempt.
 t = Thread.new do
   begin
-    client2 = server.accept( "Timed out waiting for a reconnection",
+    client2 = server.accept('Timed out waiting for a reconnection',
                             FlexNBD::MS_RETRY_DELAY_SECS + 1)
-    fail "Unexpected reconnection"
+    raise 'Unexpected reconnection'
   rescue Timeout::Error
     # expected
   end
 end
 
-client1.write_hello( :magic => :wrong )
+client1.write_hello(magic: :wrong)
 
 t.join
 

tests/acceptance/fakes/dest/hello_wrong_size.rb

@@ -18,21 +18,21 @@ t = Thread.new do
   # so it makes no sense to continue.  This means we have to invert the
   # sense of the exception.
   begin
-    client2 = server.accept( "Timed out waiting for a reconnection",
+    client2 = server.accept('Timed out waiting for a reconnection',
                             FlexNBD::MS_RETRY_DELAY_SECS + 1)
     client2.close
-    fail "Unexpected reconnection."
+    raise 'Unexpected reconnection.'
   rescue Timeout::Error
   end
 end
 
-client.write_hello( :size => :wrong )
+client.write_hello(size: :wrong)
 
 t.join
 
 # Now check that the source closed the first socket (yes, this was an
 # actual bug)
 
-fail "Didn't close socket" unless client.disconnected?
+raise "Didn't close socket" unless client.disconnected?
 
 exit 0

tests/acceptance/fakes/dest/reject_acl.rb

@@ -10,15 +10,13 @@ addr, port = *ARGV
 server = FakeDest.new(addr, port)
 server.accept.close
 
-
 begin
   server.accept
-  fail "Unexpected reconnection"
+  raise 'Unexpected reconnection'
 rescue Timeout::Error
   # expected
 end
 
 server.close
 
-
 exit(0)

tests/acceptance/fakes/dest/sigterm_after_hello.rb

@@ -9,7 +9,7 @@ include FlexNBD
 
 addr, port, pid = *ARGV
 server = FakeDest.new(addr, port)
-client = server.accept( "Timed out waiting for a connection" )
+client = server.accept('Timed out waiting for a connection')
 client.write_hello
 
 Process.kill(15, pid.to_i)

tests/acceptance/fakes/dest/write_wrong_magic.rb

@@ -1,6 +1,4 @@
 #!/usr/bin/env ruby
-# encoding: utf-8
-
 # Accept a connection, write hello, wait for a write request, read the
 # data, then write back a reply with a bad magic field.  We then
 # expect a reconnect.
@@ -15,7 +13,7 @@ client = server.accept
 client.write_hello
 req = client.read_request
 client.read_data(req[:len])
-client.write_reply( req[:handle], 0, :magic => :wrong )
+client.write_reply(req[:handle], 0, magic: :wrong)
 
 client2 = server.accept
 client.close

tests/acceptance/fakes/source/close_after_connect.rb

@@ -11,13 +11,13 @@ include FlexNBD
 
 addr, port = *ARGV
 
-FakeSource.new( addr, port, "Failed to connect" ).close
+FakeSource.new(addr, port, 'Failed to connect').close
 # Sleep to be sure we don't try to connect too soon. That wouldn't
 # be a problem for the destination, but it would prevent us from
 # determining success or failure here in the case where we try to
 # reconnect before the destination has tidied up after the first
 # thread went away.
 sleep(0.5)
-FakeSource.new( addr, port, "Failed to reconnect" ).close
+FakeSource.new(addr, port, 'Failed to reconnect').close
 
 exit 0

tests/acceptance/fakes/source/close_after_entrust.rb

@@ -11,10 +11,10 @@ include FlexNBD
 
 addr, port, srv_pid = *ARGV
 
-client = FakeSource.new( addr, port, "Timed out connecting" )
+client = FakeSource.new(addr, port, 'Timed out connecting')
 client.read_hello
 client.write_write_request(0, 8)
-client.write_data( "12345678" )
+client.write_data('12345678')
 
 # Use system "kill" rather than Process.kill because Process.kill
 # doesn't seem to work
@@ -25,12 +25,11 @@ client.close
 
 system "kill -CONT #{srv_pid}"
 
-
 sleep(0.25)
 
 begin
-  client2 = FakeSource.new( addr, port, "Expected timeout" )
+  client2 = FakeSource.new(addr, port, 'Expected timeout')
-  fail "Unexpected reconnection"
+  raise 'Unexpected reconnection'
 rescue Timeout::Error
   # expected
 end

tests/acceptance/fakes/source/close_after_entrust_reply.rb

@@ -10,10 +10,10 @@ include FlexNBD
 
 addr, port, srv_pid = *ARGV
 
-client = FakeSource.new( addr, port, "Timed out connecting" )
+client = FakeSource.new(addr, port, 'Timed out connecting')
 client.read_hello
 client.write_write_request(0, 8)
-client.write_data( "12345678" )
+client.write_data('12345678')
 
 client.write_entrust_request
 client.read_response
@@ -21,13 +21,11 @@ client.close
 
 sleep(0.25)
 
-
 begin
-  client2 = FakeSource.new( addr, port, "Expected timeout" )
+  client2 = FakeSource.new(addr, port, 'Expected timeout')
-  fail "Unexpected reconnection"
+  raise 'Unexpected reconnection'
 rescue Timeout::Error
   # expected
 end
 
 exit(0)
-

tests/acceptance/fakes/source/close_after_hello.rb

@@ -12,13 +12,12 @@ include FlexNBD
 
 addr, port = *ARGV
 
-
+client = FakeSource.new(addr, port, 'Timed out connecting.')
-client = FakeSource.new( addr, port, "Timed out connecting." )
 client.read_hello
 client.close
 
 sleep(0.2)
 
-FakeSource.new( addr, port, "Timed out reconnecting." )
+FakeSource.new(addr, port, 'Timed out reconnecting.')
 
 exit(0)

tests/acceptance/fakes/source/close_after_write.rb

@@ -1,6 +1,4 @@
 #!/usr/bin/env ruby
-# encoding: utf-8
-
 # We connect, pause the server, issue a write request, disconnect,
 # then cont the server. This ensures that our disconnect happens
 # while the server is trying to read the write data.
@@ -10,7 +8,7 @@ include FlexNBD
 
 addr, port, srv_pid = *ARGV
 
-client = FakeSource.new( addr, port, "Timed out connecting" )
+client = FakeSource.new(addr, port, 'Timed out connecting')
 client.read_hello
 
 system "kill -STOP #{srv_pid}"
@@ -24,7 +22,7 @@ system "kill -CONT #{srv_pid}"
 sleep(0.25)
 
 # ...and can we reconnect?
-client2 = FakeSource.new( addr, port, "Timed out connecting" )
+client2 = FakeSource.new(addr, port, 'Timed out connecting')
 client2.close
 
 exit(0)

tests/acceptance/fakes/source/close_after_write_data.rb

@@ -1,6 +1,4 @@
 #!/usr/bin/env ruby
-# encoding: utf-8
-
 # We connect, pause the server, issue a write request, send data,
 # disconnect, then cont the server. This ensures that our disconnect
 # happens before the server can try to write the reply.
@@ -10,13 +8,13 @@ include FlexNBD
 
 addr, port, srv_pid = *ARGV
 
-client = FakeSource.new( addr, port, "Timed out connecting" )
+client = FakeSource.new(addr, port, 'Timed out connecting')
 client.read_hello
 
 system "kill -STOP #{srv_pid}"
 
 client.write_write_request(0, 8)
-client.write_data( "12345678" )
+client.write_data('12345678')
 client.close
 
 system "kill -CONT #{srv_pid}"
@@ -27,7 +25,7 @@ system "kill -CONT #{srv_pid}"
 sleep(0.25)
 
 # ...and can we reconnect?
-client2 = FakeSource.new( addr, port, "Timed out reconnecting" )
+client2 = FakeSource.new(addr, port, 'Timed out reconnecting')
 client2.close
 
 exit(0)

tests/acceptance/fakes/source/close_mid_read.rb

@@ -8,10 +8,9 @@ include FlexNBD
 
 addr, port, srv_pid, newaddr, newport = *ARGV
 
-client = FakeSource.new( addr, port, "Timed out connecting" )
+client = FakeSource.new(addr, port, 'Timed out connecting')
 client.write_read_request(0, 8)
 client.read_raw(4)
 client.close
 
-
 exit(0)

tests/acceptance/fakes/source/connect_during_hello.rb

@@ -10,9 +10,9 @@ include FlexNBD
 
 addr, port = *ARGV
 
-client1 = FakeSource.new( addr, port, "Timed out connecting" )
+client1 = FakeSource.new(addr, port, 'Timed out connecting')
 sleep(0.25)
-client2 = FakeSource.new( addr, port, "Timed out connecting a second time" )
+client2 = FakeSource.new(addr, port, 'Timed out connecting a second time')
 
 # This is the expected source crashing after connect
 client1.close

tests/acceptance/fakes/source/connect_from_banned_ip.rb

@@ -1,6 +1,4 @@
 #!/usr/bin/env ruby
-# encoding: utf-8
-
 # We connect from a local address which should be blocked, sleep for a
 # bit, then try to read from the socket. We should get an instant EOF
 # as we've been cut off by the destination.
@@ -11,10 +9,9 @@ include FlexNBD
 
 addr, port = *ARGV
 
-client = FakeSource.new( addr, port, "Timed out connecting", "127.0.0.6" )
+client = FakeSource.new(addr, port, 'Timed out connecting', '127.0.0.6')
 sleep(0.25)
 
 rsp = client.disconnected? ? 0 : 1
 client.close
 exit(rsp)
-

tests/acceptance/fakes/source/hang_after_hello.rb

@@ -7,10 +7,10 @@
 # listening for an incoming migration.
 
 addr, port = *ARGV
-require "flexnbd/fake_source"
+require 'flexnbd/fake_source'
 include FlexNBD
 
-client = FakeSource.new( addr, port, "Timed out connecting" )
+client = FakeSource.new(addr, port, 'Timed out connecting')
 client.read_hello
 
 # Now we do two things:
@@ -25,7 +25,7 @@ kidpid = fork do
   client.close
   new_client = nil
   sleep(FlexNBD::CLIENT_MAX_WAIT_SECS + 1)
-  new_client = FakeSource.new( addr, port, "Timed out reconnecting." )
+  new_client = FakeSource.new(addr, port, 'Timed out reconnecting.')
   new_client.read_hello
   exit 0
 end

tests/acceptance/fakes/source/sigterm_after_hello.rb

@@ -9,10 +9,10 @@ include FlexNBD
 
 addr, port, pid = *ARGV
 
-client = FakeSource.new( addr, port, "Timed out connecting." )
+client = FakeSource.new(addr, port, 'Timed out connecting.')
 client.read_hello
 
-Process.kill( "TERM", pid.to_i )
+Process.kill('TERM', pid.to_i)
 
 sleep(0.2)
 client.close

tests/acceptance/fakes/source/successful_transfer.rb

@@ -9,10 +9,9 @@ include FlexNBD
 
 addr, port, srv_pid, newaddr, newport = *ARGV
 
-client  = FakeSource.new( addr, port, "Timed out connecting" )
+client = FakeSource.new(addr, port, 'Timed out connecting')
-client.send_mirror()
+client.send_mirror
 
 sleep(1)
 
-
 exit(0)

tests/acceptance/fakes/source/write_out_of_range.rb

@@ -1,6 +1,4 @@
 #!/usr/bin/env ruby
-# encoding: utf-8
-
 # Connect, read the hello then make a write request with an impossible
 # (from,len) pair.  We expect an error response, and not to be
 # disconnected.
@@ -13,20 +11,20 @@ include FlexNBD
 
 addr, port = *ARGV
 
-client = FakeSource.new( addr, port, "Timed out connecting" )
+client = FakeSource.new(addr, port, 'Timed out connecting')
 hello = client.read_hello
-client.write_write_request( hello[:size]+1, 32, "myhandle" )
+client.write_write_request(hello[:size] + 1, 32, 'myhandle')
-client.write_data("1"*32)
+client.write_data('1' * 32)
 response = client.read_response
 
-fail "Not an error" if response[:error] == 0
+raise 'Not an error' if response[:error] == 0
-fail "Wrong handle" unless "myhandle" == response[:handle]
+raise 'Wrong handle' unless response[:handle] == 'myhandle'
 
 client.write_write_request(0, 32)
-client.write_data( "2"*32 )
+client.write_data('2' * 32)
 success_response = client.read_response
 
-fail "Second write failed" unless success_response[:error] == 0
+raise 'Second write failed' unless success_response[:error] == 0
 
 client.close
 exit(0)

tests/acceptance/file_writer.rb

@@ -3,13 +3,13 @@
 #
 class FileWriter
   def initialize(filename, blocksize)
-    @fh = File.open(filename, "w+")
+    @fh = File.open(filename, 'w+')
     @blocksize = blocksize
-    @pattern = ""
+    @pattern = ''
   end
 
   def size
-    @blocksize * @pattern.split("").size
+    @blocksize * @pattern.split('').size
   end
 
   # We write in fixed block sizes, given by "blocksize"
@@ -20,8 +20,8 @@ class FileWriter
   def write(data)
     @pattern += data
 
-    data.split("").each do |code|
+    data.split('').each do |code|
-      if code == "_"
+      if code == '_'
         @fh.seek(@blocksize, IO::SEEK_CUR)
       else
         @fh.write(data(code))
@@ -31,15 +31,14 @@ class FileWriter
     self
   end
 
-
   # Returns what the data ought to be at the given offset and length
   #
   def read_original(off, len)
-    patterns = @pattern.split( "" )
+    patterns = @pattern.split('')
-    patterns.zip( (0...patterns.length).to_a ).
+    patterns.zip((0...patterns.length).to_a)
-      map { |blk, blk_off|
+            .map do |blk, blk_off|
       data(blk, blk_off)
-    }.join[off...(off+len)]
+    end.join[off...(off + len)]
   end
 
   # Read what's actually in the file
@@ -62,14 +61,14 @@ class FileWriter
 
   def data(code, at = @fh.tell)
     case code
-      when "0", "_"
+    when '0', '_'
       "\0" * @blocksize
-      when "X"
+    when 'X'
-        "X" * @blocksize
+      'X' * @blocksize
-      when "f"
+    when 'f'
-        r = ""
+      r = ''
       (@blocksize / 4).times do
-          r += [at].pack("I")
+        r += [at].pack('I')
         at += 4
       end
       r
@@ -77,51 +76,49 @@ class FileWriter
       raise "Unknown character '#{block}'"
     end
   end
-
 end
 
-if __FILE__==$0
+if $PROGRAM_NAME == __FILE__
   require 'tempfile'
   require 'test/unit'
 
   class FileWriterTest < Test::Unit::TestCase
     def test_read_original_zeros
-      Tempfile.open("test_read_original_zeros") do |tempfile|
+      Tempfile.open('test_read_original_zeros') do |tempfile|
         tempfile.close
         file = FileWriter.new(tempfile.path, 4096)
-        file.write( "0" )
+        file.write('0')
         assert_equal file.read(0, 4096), file.read_original(0, 4096)
-        assert( file.untouched?(0,4096) , "Untouched file was touched." )
+        assert(file.untouched?(0, 4096), 'Untouched file was touched.')
       end
     end
 
     def test_read_original_offsets
-      Tempfile.open("test_read_original_offsets") do |tempfile|
+      Tempfile.open('test_read_original_offsets') do |tempfile|
         tempfile.close
         file = FileWriter.new(tempfile.path, 4096)
-        file.write( "f" )
+        file.write('f')
         assert_equal file.read(0, 4096), file.read_original(0, 4096)
-        assert( file.untouched?(0,4096) , "Untouched file was touched." )
+        assert(file.untouched?(0, 4096), 'Untouched file was touched.')
       end
     end
 
     def test_file_size
-      Tempfile.open("test_file_size") do |tempfile|
+      Tempfile.open('test_file_size') do |tempfile|
         tempfile.close
         file = FileWriter.new(tempfile.path, 4096)
-        file.write( "f" )
+        file.write('f')
         assert_equal 4096, File.stat(tempfile.path).size
       end
     end
 
     def test_read_original_size
-      Tempfile.open("test_read_original_offsets") do |tempfile|
+      Tempfile.open('test_read_original_offsets') do |tempfile|
         tempfile.close
         file = FileWriter.new(tempfile.path, 4)
-        file.write( "f"*4 )
+        file.write('f' * 4)
         assert_equal 4, file.read_original(0, 4).length
       end
     end
   end
 end
-

tests/acceptance/flexnbd.rb

@@ -4,28 +4,26 @@ require 'open3'
 require 'timeout'
 require 'rexml/document'
 require 'rexml/streamlistener'
+require 'English'
 
 Thread.abort_on_exception = true
 
-
 class Executor
   attr_reader :pid
 
   def run(cmd)
-    @pid = fork do exec cmd end
+    @pid = fork { exec cmd }
   end
 end # class Executor
 
-
 class ValgrindExecutor
   attr_reader :pid
 
   def run(cmd)
-    @pid = fork do exec "valgrind --track-origins=yes --suppressions=custom.supp #{cmd}" end
+    @pid = fork { exec "valgrind --track-origins=yes --suppressions=custom.supp #{cmd}" }
   end
 end # class ValgrindExecutor
 
-
 class ValgrindKillingExecutor
   attr_reader :pid
 
@@ -34,9 +32,9 @@ class ValgrindKillingExecutor
     attr_reader :backtrace
     def initialize
       @backtrace = []
-      @what = ""
+      @what = ''
-      @kind = ""
+      @kind = ''
-      @pid = ""
+      @pid = ''
     end
 
     def add_frame
@@ -58,10 +56,8 @@ class ValgrindKillingExecutor
     def to_s
       ([@what + " (#{@kind}) in #{@pid}"] + @backtrace.map { |h| "#{h[:file]}:#{h[:line]} #{h[:fn]}" }).join("\n")
     end
-
   end # class Error
 
-
   class ErrorListener
     include REXML::StreamListener
     def initialize(killer)
@@ -74,39 +70,36 @@ class ValgrindKillingExecutor
       @text = text
     end
 
-    def tag_start(tag, attrs)
+    def tag_start(tag, _attrs)
       case tag.to_s
-      when "error"
+      when 'error'
         @found = true
-      when "frame"
+      when 'frame'
         @error.add_frame
       end
     end
 
     def tag_end(tag)
       case tag.to_s
-      when "what"
+      when 'what'
         @error.what = @text if @found
-        @text = ""
+        @text = ''
-      when "kind"
+      when 'kind'
         @error.kind = @text if @found
-      when "file"
+      when 'file'
         @error.add_file(@text) if @found
-      when "fn"
+      when 'fn'
         @error.add_fn(@text) if @found
-      when "line"
+      when 'line'
         @error.add_line(@text) if @found
-      when "error", "stack"
+      when 'error', 'stack'
-        if @found
+        @killer.call(@error) if @found
-          @killer.call( @error )
+      when 'pid'
-        end
-      when "pid"
         @error.pid = @text
       end
     end
   end # class ErrorListener
 
-
   class DebugErrorListener < ErrorListener
     def text(txt)
       print txt
@@ -124,47 +117,41 @@ class ValgrindKillingExecutor
     end
   end
 
-
   def initialize
     @pid = nil
   end
 
   def run(cmd)
     @io_r, io_w = IO.pipe
-    @pid = fork do exec( "valgrind --suppressions=custom.supp --xml=yes --xml-fd=#{io_w.fileno} " + cmd ) end
+    @pid = fork { exec("valgrind --suppressions=custom.supp --xml=yes --xml-fd=#{io_w.fileno} " + cmd) }
     launch_watch_thread(@pid, @io_r)
     @pid
   end
 
-
   def call(err)
-    $stderr.puts "*"*72
+    warn '*' * 72
-    $stderr.puts "* Valgrind error spotted:"
+    warn '* Valgrind error spotted:'
-    $stderr.puts err.to_s.split("\n").map{|s| "  #{s}"}
+    warn err.to_s.split("\n").map { |s| "  #{s}" }
-    $stderr.puts "*"*72
+    warn '*' * 72
-    Process.kill( "KILL", @pid )
+    Process.kill('KILL', @pid)
     exit(1)
   end
 
-
   private
 
   def pick_listener
     ENV['DEBUG'] ? DebugErrorListener : ErrorListener
   end
 
-  def launch_watch_thread(pid, io_r)
+  def launch_watch_thread(_pid, io_r)
     Thread.start do
       io_source = REXML::IOSource.new(io_r)
       listener = pick_listener.new(self)
       REXML::Document.parse_stream(io_source, listener)
     end
   end
-
-
 end # class ValgrindExecutor
 
-
 module FlexNBD
   # Noddy test class to exercise FlexNBD from the outside for testing.
   #
@@ -189,12 +176,11 @@ module FlexNBD
             end
     end
 
-
     def build_debug_opt
       if @do_debug
-        "--verbose"
+        '--verbose'
       else
-        "--quiet"
+        '--quiet'
       end
     end
 
@@ -209,20 +195,19 @@ module FlexNBD
       @ctrl = "/tmp/.flexnbd.ctrl.#{Time.now.to_i}.#{rand}"
       @ip = ip
       @port = port
+      @pid = @wait_thread = nil
       @kill = []
       @prefetch_proxy = false
     end
 
-
     def debug?
       !!@do_debug
     end
 
     def debug(msg)
-      $stderr.puts msg if debug?
+      warn msg if debug?
     end
 
-
     def serve_cmd(file, acl)
       "#{bin} serve "\
         "--addr #{ip} "\
@@ -233,7 +218,6 @@ module FlexNBD
         "#{acl.join(' ')}"
     end
 
-
     def listen_cmd(file, acl)
       "#{bin} listen "\
         "--addr #{ip} "\
@@ -250,11 +234,10 @@ module FlexNBD
         "--port #{port} "\
         "--conn-addr #{connect_ip} "\
         "--conn-port #{connect_port} "\
-        "#{prefetch_proxy ? "--cache " : ""}"\
+        "#{prefetch_proxy ? '--cache ' : ''}"\
         "#{@debug}"
     end
 
-
     def read_cmd(offset, length)
       "#{bin} read "\
         "--addr #{ip} "\
@@ -264,7 +247,6 @@ module FlexNBD
         "--size #{length}"
     end
 
-
     def write_cmd(offset, data)
       "#{bin} write "\
         "--addr #{ip} "\
@@ -274,7 +256,6 @@ module FlexNBD
         "--size #{data.length}"
     end
 
-
     def base_mirror_opts(dest_ip, dest_port)
       "--addr #{dest_ip} "\
         "--port #{dest_port} "\
@@ -283,7 +264,7 @@ module FlexNBD
 
     def unlink_mirror_opts(dest_ip, dest_port)
       "#{base_mirror_opts(dest_ip, dest_port)} "\
-        "--unlink "
+        '--unlink '
     end
 
     def base_mirror_cmd(opts)
@@ -316,46 +297,52 @@ module FlexNBD
       "#{@bin} acl " \
         "--sock #{ctrl} "\
         "#{@debug} "\
-        "#{acl.join " "}"
+        "#{acl.join ' '}"
     end
 
-
     def run_serve_cmd(cmd)
-      File.unlink(ctrl) if File.exists?(ctrl)
+      File.unlink(ctrl) if File.exist?(ctrl)
       debug(cmd)
 
       @pid = @executor.run(cmd)
 
-      while !File.socket?(ctrl)
+      until File.socket?(ctrl)
         pid, status = Process.wait2(@pid, Process::WNOHANG)
         raise "server did not start (#{cmd})" if pid
         sleep 0.1
       end
 
-
       start_wait_thread(@pid)
       at_exit { kill }
     end
     private :run_serve_cmd
 
-
     def serve(file, *acl)
       cmd = serve_cmd(file, acl)
       run_serve_cmd(cmd)
-      sleep( 0.2 ) until File.exists?( ctrl )
+      sleep(0.2) until File.exist?(ctrl)
     end
 
-
     def listen(file, *acl)
       run_serve_cmd(listen_cmd(file, acl))
     end
 
     def tcp_server_open?
       # raises if the other side doesn't accept()
-      sock = TCPSocket.new(ip, port) rescue nil
+      sock = begin
+               TCPSocket.new(ip, port)
+             rescue StandardError
+               nil
+             end
 
       success = !!sock
-      ( sock.close rescue nil) if sock
+      if sock
+        (begin
+            sock.close
+          rescue StandardError
+            nil
+          end)
+      end
       success
     end
 
@@ -375,27 +362,25 @@ module FlexNBD
       at_exit { kill }
     end
 
-
     def start_wait_thread(pid)
       @wait_thread = Thread.start do
         _, status = Process.waitpid2(pid)
 
         if @kill
           if status.signaled?
-            fail "flexnbd quit with a bad signal: #{status.inspect}" unless
+            raise "flexnbd quit with a bad signal: #{status.inspect}" unless
             @kill.include? status.termsig
           else
-            fail "flexnbd quit with a bad status: #{status.inspect}" unless
+            raise "flexnbd quit with a bad status: #{status.inspect}" unless
             @kill.include? status.exitstatus
           end
         else
-          $stderr.puts "flexnbd #{self.pid} quit"
+          warn "flexnbd #{self.pid} quit"
-          fail "flexnbd #{self.pid} quit early with status #{status.to_i}"
+          raise "flexnbd #{self.pid} quit early with status #{status.to_i}"
         end
       end
     end
 
-
     def can_die(*status)
       status = [0] if status.empty?
       @kill += status
@@ -407,7 +392,7 @@ module FlexNBD
       can_die(1)
       if @pid
         begin
-          Process.kill("INT", @pid)
+          Process.kill('INT', @pid)
         rescue Errno::ESRCH => e
           # already dead.  Presumably this means it went away after a
           # can_die() call.
@@ -423,7 +408,7 @@ module FlexNBD
       IO.popen(cmd) do |fh|
         return fh.read
       end
-      raise IOError.new "NBD read failed" unless $?.success?
+      raise IOError, 'NBD read failed' unless $CHILD_STATUS.success?
       out
     end
 
@@ -431,27 +416,24 @@ module FlexNBD
       cmd = write_cmd(offset, data)
       debug(cmd)
 
-      IO.popen(cmd, "w") do |fh|
+      IO.popen(cmd, 'w') do |fh|
         fh.write(data)
       end
-      raise IOError.new "NBD write failed" unless $?.success?
+      raise IOError, 'NBD write failed' unless $CHILD_STATUS.success?
       nil
     end
 
-
     def join
       @wait_thread.join
     end
 
-
+    def mirror_unchecked(dest_ip, dest_port, _bandwidth = nil, _action = nil, timeout = nil)
-    def mirror_unchecked( dest_ip, dest_port, bandwidth=nil, action=nil, timeout=nil )
       cmd = mirror_cmd(dest_ip, dest_port)
       debug(cmd)
 
       maybe_timeout(cmd, timeout)
     end
 
-
     def mirror_unlink(dest_ip, dest_port, timeout = nil)
       cmd = mirror_unlink_cmd(dest_ip, dest_port)
       debug(cmd)
@@ -459,11 +441,11 @@ module FlexNBD
       maybe_timeout(cmd, timeout)
     end
 
-
     def maybe_timeout(cmd, timeout = nil)
-      stdout, stderr = "",""
+      stdout = ''
+      stderr = ''
       stat = nil
-      run = Proc.new do
+      run = proc do
         # Ruby 1.9 changed the popen3 api. instead of 3 args, the block
         # gets 4. Not only that, but it no longer sets $?, so we have to
         # go elsewhere for the process' exit status.
@@ -473,7 +455,7 @@ module FlexNBD
           stderr.replace io_err.read
           stat = maybe_thr.value if maybe_thr
         end
-        stat ||= $?
+        stat ||= $CHILD_STATUS
       end
 
       if timeout
@@ -485,16 +467,13 @@ module FlexNBD
       [stdout, stderr, stat]
     end
 
-
     def mirror(dest_ip, dest_port, bandwidth = nil, action = nil)
       stdout, stderr, status = mirror_unchecked(dest_ip, dest_port, bandwidth, action)
-      raise IOError.new( "Migrate command failed\n" + stderr) unless status.success?
+      raise IOError, "Migrate command failed\n" + stderr unless status.success?
 
       stdout
     end
 
-
-
     def break(timeout = nil)
       cmd = break_cmd
       debug(cmd)
@@ -502,7 +481,6 @@ module FlexNBD
       maybe_timeout(cmd, timeout)
     end
 
-
     def acl(*acl)
       cmd = acl_cmd(*acl)
       debug(cmd)
@@ -510,9 +488,8 @@ module FlexNBD
       maybe_timeout(cmd, 2)
     end
 
-
     def status(timeout = nil)
-      cmd = status_cmd()
+      cmd = status_cmd
       debug(cmd)
 
       o, e = maybe_timeout(cmd, timeout)
@@ -520,50 +497,43 @@ module FlexNBD
       [parse_status(o), e]
     end
 
-
     def launched?
       !!@pid
     end
 
-
     def paused
-      Process.kill( "STOP", @pid )
+      Process.kill('STOP', @pid)
       yield
     ensure
-      Process.kill( "CONT", @pid )
+      Process.kill('CONT', @pid)
     end
 
-
     protected
+
     def control_command(*args)
-      raise "Server not running" unless @pid
+      raise 'Server not running' unless @pid
       args = args.compact
       UNIXSocket.open(@ctrl) do |u|
         u.write(args.join("\n") + "\n")
-        code, message = u.readline.split(": ", 2)
+        code, message = u.readline.split(': ', 2)
         return [code, message]
       end
     end
 
-
     def parse_status(status)
       hsh = {}
 
-      status.split(" ").each do |part|
+      status.split(' ').each do |part|
         next if part.strip.empty?
-        a,b = part.split("=")
+        a, b = part.split('=')
         b.strip!
-        b = true if b == "true"
+        b = true if b == 'true'
-        b = false if b == "false"
+        b = false if b == 'false'
 
         hsh[a.strip] = b
       end
 
       hsh
     end
-
-
   end
-
 end
-

tests/acceptance/flexnbd/constants.rb

@@ -1,10 +1,7 @@
-# encoding: utf-8
-
 module FlexNBD
-
   def self.binary(str)
     if str.respond_to? :force_encoding
-      str.force_encoding "ASCII-8BIT"
+      str.force_encoding 'ASCII-8BIT'
     else
       str
     end
@@ -13,36 +10,33 @@ module FlexNBD
   # eeevil is his one and only name...
   def self.read_constants
     parents = []
-    current = File.expand_path(".")
+    current = File.expand_path('.')
-    while current != "/"
+    while current != '/'
       parents << current
-      current = File.expand_path( File.join( current, ".." ) )
+      current = File.expand_path(File.join(current, '..'))
     end
 
     source_root = parents.find do |dirname|
-      File.directory?( File.join( dirname, "src" ) )
+      File.directory?(File.join(dirname, 'src'))
     end
 
-    fail "No source root!" unless source_root
+    raise 'No source root!' unless source_root
 
-    headers = Dir[File.join( source_root, "src", "{common,proxy,server}","*.h" ) ]
+    headers = Dir[File.join(source_root, 'src', '{common,proxy,server}', '*.h')]
 
     headers.each do |header_filename|
       txt_lines = File.readlines(header_filename)
       txt_lines.each do |line|
         if line =~ /^#\s*define\s+([A-Z0-9_]+)\s+(\d+)\s*$/
           # Bodge until I can figure out what to do with #ifdefs
-          const_set($1, $2.to_i) unless const_defined?( $1 )
+          const_set(Regexp.last_match(1), Regexp.last_match(2).to_i) unless const_defined?(Regexp.last_match(1))
+        end
       end
     end
   end
 
-  end
+  read_constants
-
-  read_constants()
 
   REQUEST_MAGIC = binary("\x25\x60\x95\x13") unless defined?(REQUEST_MAGIC)
   REPLY_MAGIC = binary("\x67\x44\x66\x98") unless defined?(REPLY_MAGIC)
-
 end # module FlexNBD
-

tests/acceptance/flexnbd/fake_dest.rb

@@ -1,21 +1,18 @@
-# encoding: utf-8
-
 require 'socket'
 require 'timeout'
+require 'io/wait' # For IO#nread
 
 require 'flexnbd/constants'
 
 module FlexNBD
   class FakeDest
-
     class Client
       def initialize(sock)
         @sock = sock
       end
 
-
       def write_hello(opts = {})
-        @sock.write( "NBDMAGIC" )
+        @sock.write('NBDMAGIC')
 
         if opts[:magic] == :wrong
           write_rand(@sock, 8)
@@ -32,13 +29,11 @@ module FlexNBD
         @sock.write("\x00" * 128)
       end
 
-
       def write_rand(sock, len)
-        len.times do sock.write( rand(256).chr ) end
+        len.times { sock.write(rand(256).chr) }
       end
 
-
+      def read_request
-      def read_request()
         req = @sock.read(28)
 
         magic_s  = req[0...4]
@@ -48,11 +43,11 @@ module FlexNBD
         len_s    = req[24...28]
 
         {
-          :magic  => magic_s,
+          magic: magic_s,
-          :type   => type_s.unpack("N").first,
+          type: type_s.unpack('N').first,
-          :handle => handle_s,
+          handle: handle_s,
-          :from   => self.class.parse_be64( from_s ),
+          from: self.class.parse_be64(from_s),
-          :len    => len_s.unpack( "N").first
+          len: len_s.unpack('N').first
         }
       end
 
@@ -60,15 +55,17 @@ module FlexNBD
         write_reply(handle, 1)
       end
 
+      def nread
+        @sock.nread
+      end
+
       def disconnected?
-        begin
         Timeout.timeout(2) do
-            @sock.read(1) == nil
+          @sock.read(1).nil?
         end
       rescue Timeout::Error
         return false
       end
-      end
 
       def write_reply(handle, err = 0, opts = {})
         if opts[:magic] == :wrong
@@ -77,16 +74,14 @@ module FlexNBD
           @sock.write(::FlexNBD::REPLY_MAGIC)
         end
 
-        @sock.write( [err].pack("N") )
+        @sock.write([err].pack('N'))
         @sock.write(handle)
       end
 
-
       def close
         @sock.close
       end
 
-
       def read_data(len)
         @sock.read(len)
       end
@@ -95,25 +90,27 @@ module FlexNBD
         @sock.write(len)
       end
 
+      def getsockopt(level, optname)
+        @sock.getsockopt(level, optname)
+      end
 
       def self.parse_be64(str)
         raise "String is the wrong length: 8 bytes expected (#{str.length} received)" unless
           str.length == 8
 
-        top, bottom = str.unpack("NN")
+        top, bottom = str.unpack('NN')
         (top << 32) + bottom
       end
 
-
       def receive_mirror(opts = {})
-        write_hello()
+        write_hello
         loop do
           req = read_request
           case req[:type]
           when 1
             read_data(req[:len])
             write_reply(req[:handle])
-          when 65536
+          when 65_536
             write_reply(req[:handle], opts[:err] == :entrust ? 1 : 0)
             break
           else
@@ -129,16 +126,13 @@ module FlexNBD
           raise "Not a disconnect: #{req.inspect}"
         end
       end
-
     end # class Client
 
-
     def initialize(addr, port)
       @sock = TCPServer.new(addr, port)
     end
 
-
+    def accept(err_msg = 'Timed out waiting for a connection', timeout = 5)
-    def accept( err_msg = "Timed out waiting for a connection", timeout = 5)
       client_sock = nil
 
       begin
@@ -146,7 +140,7 @@ module FlexNBD
           client_sock = @sock.accept
         end
       rescue Timeout::Error
-        raise Timeout::Error.new(err_msg)
+        raise Timeout::Error, err_msg
       end
 
       client_sock
@@ -154,13 +148,8 @@ module FlexNBD
       Client.new(client_sock)
     end
 
-
     def close
       @sock.close
     end
-
-
-
   end # module FakeDest
 end # module FlexNBD
-

tests/acceptance/flexnbd/fake_source.rb

@@ -1,12 +1,9 @@
-# encoding: utf-8
-
 require 'socket'
-require "timeout"
+require 'timeout'
 require 'flexnbd/constants'
 
 module FlexNBD
   class FakeSource
-
     def initialize(addr, port, err_msg, source_addr = nil, source_port = 0)
       timing_out(2, err_msg) do
         begin
@@ -16,75 +13,76 @@ module FlexNBD
                     TCPSocket.new(addr, port)
                   end
         rescue Errno::ECONNREFUSED
-          $stderr.puts "Connection refused, retrying"
+          warn 'Connection refused, retrying'
           sleep(0.2)
           retry
         end
       end
     end
 
-
     def close
       @sock.close
     end
 
-
+    def read_hello
-    def read_hello()
       timing_out(::FlexNBD::MS_HELLO_TIME_SECS,
-                  "Timed out waiting for hello." ) do
+                 'Timed out waiting for hello.') do
-        fail "No hello." unless (hello = @sock.read( 152 )) &&
+        raise 'No hello.' unless (hello = @sock.read(152)) &&
                                  hello.length == 152
 
-        magic_s = hello[0..7]
+        passwd_s = hello[0..7]
-        ignore_s= hello[8..15]
+        magic    = hello[8..15].unpack('Q>').first
-        size_s  = hello[16..23]
+        size     = hello[16..23].unpack('Q>').first
+        flags    = hello[24..27].unpack('L>').first
+        reserved = hello[28..-1]
 
-        size_h, size_l = size_s.unpack("NN")
+        return { passwd: passwd_s, magic: magic, size: size, flags: flags, reserved: reserved }
-        size = (size_h << 32) + size_l
-
-        return { :magic => magic_s, :size => size }
       end
     end
 
-
+    def send_request(type, handle = 'myhandle', from = 0, len = 0, magic = REQUEST_MAGIC, flags = 0)
-    def send_request( type, handle="myhandle", from=0, len=0, magic=REQUEST_MAGIC )
+      raise 'Bad handle' unless handle.length == 8
-      fail "Bad handle" unless handle.length == 8
 
       @sock.write(magic)
-      @sock.write( [type].pack( 'N' ) )
+      @sock.write([flags].pack('n'))
+      @sock.write([type].pack('n'))
       @sock.write(handle)
       @sock.write([n64(from)].pack('q'))
       @sock.write([len].pack('N'))
     end
 
-
+    def write_write_request(from, len, handle = 'myhandle')
-    def write_write_request( from, len, handle="myhandle" )
       send_request(1, handle, from, len)
     end
 
-
+    def write_write_request_with_fua(from, len, handle = 'myhandle')
-    def write_entrust_request( handle="myhandle" )
+      send_request(1, handle, from, len, REQUEST_MAGIC, 1)
-      send_request( 65536, handle )
     end
 
-    def write_disconnect_request( handle="myhandle" )
+    def write_flush_request(handle = 'myhandle')
+      send_request(3, handle, 0, 0)
+    end
+
+    def write_entrust_request(handle = 'myhandle')
+      send_request(65_536, handle)
+    end
+
+    def write_disconnect_request(handle = 'myhandle')
       send_request(2, handle)
     end
 
-    def write_read_request( from, len, handle="myhandle" )
+    def write_read_request(from, len, _handle = 'myhandle')
-      send_request( 0, "myhandle", from, len )
+      send_request(0, 'myhandle', from, len)
     end
 
-
     def write_data(data)
       @sock.write(data)
     end
 
-
     # Handy utility
     def read(from, len)
-      timing_out( 2, "Timed out reading" ) do
+      timing_out(2, 'Timed out reading') do
-        send_request( 0, "myhandle", from, len )
+        send_request(0, 'myhandle', from, len)
         read_raw(len)
       end
     end
@@ -94,19 +92,26 @@ module FlexNBD
     end
 
     def send_mirror
-      read_hello()
+      read_hello
-      write( 0, "12345678" )
+      write(0, '12345678')
-      read_response()
+      read_response
-      write_disconnect_request()
+      write_disconnect_request
-      close()
+      close
     end
 
-
     def write(from, data)
       write_write_request(from, data.length)
       write_data(data)
     end
 
+    def write_with_fua(from, data)
+      write_write_request_with_fua(from, data.length)
+      write_data(data)
+    end
+
+    def flush
+      write_flush_request
+    end
 
     def read_response
       magic = @sock.read(4)
@@ -114,30 +119,26 @@ module FlexNBD
       handle = @sock.read(8)
 
       {
-        :magic => magic,
+        magic: magic,
-        :error => error_s.unpack("N").first,
+        error: error_s.unpack('N').first,
-        :handle => handle
+        handle: handle
       }
     end
 
-
     def disconnected?
       result = nil
-      Timeout.timeout( 2 ) { result = ( @sock.read(1) == nil ) }
+      Timeout.timeout(2) { result = @sock.read(1).nil? }
       result
     end
 
-
     def timing_out(time, msg)
-      begin
       Timeout.timeout(time) do
         yield
       end
     rescue Timeout::Error
-        $stderr.puts msg
+      warn msg
       exit 1
     end
-    end
 
     private
 
@@ -154,7 +155,5 @@ module FlexNBD
         ((b & 0x000000000000ff00) << 40) |
         ((b & 0x00000000000000ff) << 56)
     end
-
   end # class FakeSource
 end # module FlexNBD
-

tests/acceptance/ld_preload.rb

@@ -0,0 +1,55 @@
+require 'tempfile'
+
+#
+# LdPreload is a little wrapper for using LD_PRELOAD loggers to pick up system
+# calls when testing flexnbd.
+#
+module LdPreload
+  #
+  # This takes an object name, sets up a temporary log file, whose name is
+  # recorded in the environment as OUTPUT_obj_name, where obj_name is the
+  # name of the preload module to build and load.
+  def with_ld_preload(obj_name)
+    @ld_preload_logs ||= {}
+    flunk 'Can only load a preload module once!' if @ld_preload_logs[obj_name]
+
+    system("make -C ld_preloads/ #{obj_name}.o > /dev/null") ||
+      flunk("Failed to build object #{obj_name}")
+    orig_env = ENV['LD_PRELOAD']
+    ENV['LD_PRELOAD'] = [orig_env, File.expand_path("./ld_preloads/#{obj_name}.o")].compact.join(' ')
+
+    # Open the log, and stick it in a hash
+    @ld_preload_logs[obj_name] = Tempfile.new(obj_name)
+    ENV['OUTPUT_' + obj_name] = @ld_preload_logs[obj_name].path
+
+    yield
+  ensure
+    if @ld_preload_logs[obj_name]
+      @ld_preload_logs[obj_name].close
+      @ld_preload_logs.delete(obj_name)
+    end
+    ENV['LD_PRELOAD'] = orig_env
+  end
+
+  def read_ld_preload_log(obj_name)
+    lines = []
+    lines << @ld_preload_logs[obj_name].readline.chomp until
+      @ld_preload_logs[obj_name].eof?
+    lines
+  end
+
+  #
+  # The next to methods assume the log file has one entry per line, and that
+  # each entry is a series of values separated by colons.
+  #
+  def parse_ld_preload_logs(obj_name)
+    read_ld_preload_log(obj_name).map do |l|
+      l.split(':').map { |i| i =~ /^\d+$/ ? i.to_i : i }
+    end
+  end
+
+  def assert_func_call(loglines, args, msg)
+    re = Regexp.new('^' + args.join(':'))
+    assert(loglines.any? { |l| l.match(re) }, msg)
+  end
+end

tests/acceptance/ld_preloads/Makefile

@@ -0,0 +1,13 @@
+
+SRC := $(wildcard *.c)
+OBJS := $(SRC:%.c=%.o)
+
+all: $(OBJS)
+
+clean:
+	$(RM) $(OBJS)
+
+%.o: %.c
+	gcc -shared -fPIC -ldl -o $@ $<
+
+.PHONY: all clean

tests/acceptance/ld_preloads/msync_logger.c

@@ -0,0 +1,33 @@
+#define _GNU_SOURCE
+
+#include <dlfcn.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+
+typedef int (*real_msync_t)(void *addr, size_t length, int flags);
+
+int real_msync(void *addr, size_t length, int flags) {
+  return ((real_msync_t)dlsym(RTLD_NEXT, "msync"))(addr, length, flags);
+}
+
+/*
+ * Noddy LD_PRELOAD wrapper to catch msync calls, and log them to a file.
+ */
+
+int msync(void *addr, size_t length, int flags) {
+  FILE *fd;
+  char *fn;
+  int retval;
+  
+  retval = real_msync(addr, length, flags);
+  
+  fn = getenv("OUTPUT_msync_logger");
+  if ( fn != NULL ) {
+    fd = fopen(fn,"a");
+    fprintf(fd,"msync:%d:%i:%i:%i\n", addr, length, flags, retval);
+    fclose(fd);
+  }
+
+  return retval;
+}

tests/acceptance/ld_preloads/setsockopt_logger.c

@@ -0,0 +1,38 @@
+#define _GNU_SOURCE
+
+#include <dlfcn.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/socket.h>
+
+/*
+ * Noddy LD_PRELOAD wrapper to catch setsockopt calls, and log them to a file.
+ */
+
+typedef int (*real_setsockopt_t)(int sockfd, int level, int optname, const void *optval, socklen_t optlen);
+
+int real_setsockopt(int sockfd, int level, int optname, const void *optval, socklen_t optlen)
+{
+  return ((real_setsockopt_t)dlsym(RTLD_NEXT, "setsockopt"))(sockfd, level, optname, optval, optlen);
+}
+
+int setsockopt(int sockfd, int level, int optname, const void *optval, socklen_t optlen)
+{
+  FILE *fd;
+  char *fn;
+  int retval;
+  
+  retval = real_setsockopt(sockfd, level, optname, optval, optlen);
+  fn = getenv("OUTPUT_setsockopt_logger");
+
+  /* 
+   * Only interested in catching non-null 4-byte (integer) values 
+   */
+  if ( fn != NULL && optval != NULL && optlen == 4) {
+    fd = fopen(fn,"a");
+    fprintf(fd,"setsockopt:%i:%i:%i:%i:%i\n", sockfd, level, optname, *(int *)optval, retval);
+    fclose(fd);
+  }
+
+  return retval;
+}

tests/acceptance/nbd_scenarios

@@ -1,6 +1,6 @@
 #!/usr/bin/ruby
 
-test_files = Dir[File.dirname( __FILE__ ) + "/test*.rb"]
+test_files = Dir[File.dirname(__FILE__) + '/test*.rb']
 for filename in test_files
   require filename
 end

tests/acceptance/proxy_tests.rb

@@ -1,8 +1,12 @@
-# encoding: utf-8
+
 require 'flexnbd/fake_source'
 require 'flexnbd/fake_dest'
+require 'ld_preload'
 
 module ProxyTests
+
+  include LdPreload
+
   def b
     "\xFF".b
   end
@@ -13,14 +17,17 @@ module ProxyTests
     @env.nbd2.can_die(0)
     client = FlexNBD::FakeSource.new(@env.ip, @env.port2, "Couldn't connect to proxy")
     begin
-
       result = client.read_hello
-      assert_equal "NBDMAGIC", result[:magic]
+      assert_equal 'NBDMAGIC', result[:passwd]
       assert_equal override_size || @env.file1.size, result[:size]
 
       yield client
     ensure
-      client.close rescue nil
+      begin
+        client.close
+      rescue StandardError
+        nil
+      end
     end
   end
 
@@ -32,11 +39,11 @@ module ProxyTests
     with_proxied_client do |client|
       (0..3).each do |n|
         offset = n * 4096
-        client.write_read_request(offset, 4096, "myhandle")
+        client.write_read_request(offset, 4096, 'myhandle')
         rsp = client.read_response
 
         assert_equal ::FlexNBD::REPLY_MAGIC, rsp[:magic]
-        assert_equal "myhandle", rsp[:handle]
+        assert_equal 'myhandle', rsp[:handle]
         assert_equal 0, rsp[:error]
 
         orig_data = @env.file1.read(offset, 4096)
@@ -59,7 +66,7 @@ module ProxyTests
         rsp = client.read_response
 
         assert_equal FlexNBD::REPLY_MAGIC, rsp[:magic]
-        assert_equal "myhandle", rsp[:handle]
+        assert_equal 'myhandle', rsp[:handle]
         assert_equal 0, rsp[:error]
 
         data = @env.file1.read(offset, 4096)
@@ -69,6 +76,20 @@ module ProxyTests
     end
   end
 
+  def test_write_request_past_end_of_disc_returns_to_client
+    with_proxied_client do |client|
+      n = 1000
+      offset = n * 4096
+      client.write(offset, b * 4096)
+      rsp = client.read_response
+
+      assert_equal FlexNBD::REPLY_MAGIC, rsp[:magic]
+      assert_equal 'myhandle', rsp[:handle]
+      # NBD protocol say ENOSPC (28) in this situation
+      assert_equal 28, rsp[:error]
+    end
+  end
+
   def make_fake_server
     server = FlexNBD::FakeDest.new(@env.ip, @env.port1)
     @server_up = true
@@ -114,37 +135,41 @@ module ProxyTests
       sc2.write_data(b * 4096)
 
       # Check it to make sure it's correct
-      rsp = timeout(15) { client.read_response }
+      rsp = Timeout.timeout(15) { client.read_response }
       assert_equal ::FlexNBD::REPLY_MAGIC, rsp[:magic]
       assert_equal 0, rsp[:error]
       assert_equal req1[:handle], rsp[:handle]
 
       data = client.read_raw(4096)
-      assert_equal( (b * 4096), data, "Wrong data returned" )
+      assert_equal((b * 4096), data, 'Wrong data returned')
 
       sc2.close
       server.close
     end
-
   end
 
   def test_write_request_retried_when_upstream_dies_partway
     maker = make_fake_server
 
+    with_ld_preload('setsockopt_logger') do
       with_proxied_client(4096) do |client|
         server, sc1 = maker.value
 
         # Send the read request to the proxy
         client.write(0, (b * 4096))
 
-      # ensure we're given the read request
+        # ensure we're given the write request
         req1 = sc1.read_request
         assert_equal ::FlexNBD::REQUEST_MAGIC, req1[:magic]
         assert_equal ::FlexNBD::REQUEST_WRITE, req1[:type]
         assert_equal 0, req1[:from]
         assert_equal 4096, req1[:len]
         data1 = sc1.read_data(4096)
-      assert_equal( ( b * 4096 ), data1, "Data not proxied successfully" )
+        assert_equal((b * 4096), data1, 'Data not proxied successfully')
+
+        # Read the setsockopt logs, so we can check that TCP_NODELAY is re-set
+        # later
+        read_ld_preload_log('setsockopt_logger')
 
         # Kill the server again, now we're sure the read request has been sent once
         sc1.close
@@ -163,32 +188,109 @@ module ProxyTests
         sc2.write_reply(req2[:handle])
 
         # Check it to make sure it's correct
-      rsp = timeout(15) { client.read_response }
+        rsp = Timeout.timeout(15) { client.read_response }
         assert_equal ::FlexNBD::REPLY_MAGIC, rsp[:magic]
         assert_equal 0, rsp[:error]
         assert_equal req1[:handle], rsp[:handle]
 
         sc2.close
         server.close
+
+        # Check TCP_NODELAY was set on the upstream socket
+        log = read_ld_preload_log('setsockopt_logger')
+        assert_func_call(log,
+                         ['setsockopt', 3,
+                          Socket::SOL_TCP, Socket::TCP_NODELAY, 1, 0],
+                         'TCP_NODELAY not set on upstream fd 3')
+      end
+    end
+  end
+
+  def test_write_request_retried_when_upstream_times_out_during_write_phase
+    ENV['FLEXNBD_UPSTREAM_TIMEOUT'] = '4'
+    maker = make_fake_server
+
+    with_ld_preload('setsockopt_logger') do
+      with_proxied_client(4096) do |client|
+        server, sc1 = maker.value
+
+        # Guess an approprate request size, based on the send buffer size.
+        sz = sc1.getsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF).int * 4
+        data1 = (b * sz)
+
+        # Send the read request to the proxy
+        client.write(0, data1)
+
+        # ensure we're given the write request
+        req1 = sc1.read_request
+        assert_equal ::FlexNBD::REQUEST_MAGIC, req1[:magic]
+        assert_equal ::FlexNBD::REQUEST_WRITE, req1[:type]
+        assert_equal 0, req1[:from]
+        assert_equal data1.size, req1[:len]
+
+        # We do not read it at this point, as we want the proxy to be waiting
+        # in the WRITE_UPSTREAM state.
+
+        # Need to sleep longer than the timeout set above
+        sleep 5
+
+        # Check the number of bytes that can be read from the socket without
+        # blocking.  If this equal to the size of the original request, then
+        # the whole request has been buffered.  If this is the case, then the
+        # proxy will not time-out in the WRITE_UPSTREAM statem which is what
+        # we're trying to test.
+        assert sc1.nread < sz, 'Request from proxy completely buffered. Test is useless'
+
+        # Kill the server now that the timeout has happened.
+        sc1.close
+
+        # We expect the proxy to reconnect without our client doing anything.
+        sc2 = server.accept
+        sc2.write_hello
+
+        # And once reconnected, it should resend an identical request.
+        req2 = sc2.read_request
+        assert_equal req1, req2
+
+        # And now lets read the data to make sure we get it all.
+        data2 = sc2.read_data(req2[:len])
+        assert_equal data1, data2
+
+        sc2.close
+        server.close
+      end
     end
   end
 
   def test_only_one_client_can_connect_to_proxy_at_a_time
-    with_proxied_client do |client|
+    with_proxied_client do |_client|
-
       c2 = nil
       assert_raises(Timeout::Error) do
-        timeout(1) do
+        Timeout.timeout(1) do
           c2 = FlexNBD::FakeSource.new(@env.ip, @env.port2, "Couldn't connect to proxy (2)")
           c2.read_hello
         end
       end
-      c2.close rescue nil if c2
+      if c2
+        begin
+          c2.close
+        rescue StandardError
+          nil
+        end
+      end
+    end
   end
 
-
+  def test_maximum_write_request_size
+    # Defined in src/common/nbdtypes.h
+    nbd_max_block_size = 32 * 1024 * 1024
+    @env.writefile1('0' * 40 * 1024)
+    with_proxied_client do |client|
+      # This will crash with EPIPE if the proxy dies.
+      client.write(0, b * nbd_max_block_size)
+      rsp = client.read_response
+      assert_equal FlexNBD::REPLY_MAGIC, rsp[:magic]
+      assert_equal 0, rsp[:error]
+    end
   end
-
 end
-
-

tests/acceptance/test_dest_error_handling.rb

@@ -1,13 +1,10 @@
-# encoding: utf-8
-
 require 'test/unit'
 require 'environment'
 
 class TestDestErrorHandling < Test::Unit::TestCase
-
   def setup
     @env = Environment.new
-    @env.writefile1( "0" * 4 )
+    @env.writefile1('0' * 4)
     @env.listen1
   end
 
@@ -15,89 +12,77 @@ class TestDestErrorHandling  < Test::Unit::TestCase
     @env.cleanup
   end
 
-
   def test_hello_blocked_by_disconnect_causes_error_not_fatal
-    run_fake( "source/close_after_connect" )
+    run_fake('source/close_after_connect')
     assert_no_control
   end
 
-=begin
+  #   # This is disabled while CLIENT_MAX_WAIT_SECS is removed
-  # This is disabled while CLIENT_MAX_WAIT_SECS is removed
+  #   def test_hello_goes_astray_causes_timeout_error
-  def test_hello_goes_astray_causes_timeout_error
+  #     run_fake( "source/hang_after_hello" )
-    run_fake( "source/hang_after_hello" )
+  #     assert_no_control
-    assert_no_control
+  #   end
-  end
-=end
 
   def test_sigterm_has_bad_exit_status
     @env.nbd1.can_die(1)
-    run_fake( "source/sigterm_after_hello" )
+    run_fake('source/sigterm_after_hello')
   end
 
   def test_disconnect_after_hello_causes_error_not_fatal
-    run_fake( "source/close_after_hello" )
+    run_fake('source/close_after_hello')
     assert_no_control
   end
 
-
   def test_partial_read_causes_error
-    run_fake( "source/close_mid_read" )
+    run_fake('source/close_mid_read')
   end
 
   def test_double_connect_during_hello
-    run_fake( "source/connect_during_hello" )
+    run_fake('source/connect_during_hello')
   end
 
-
   def test_acl_rejection
-    @env.acl1("127.0.0.1")
+    @env.acl1('127.0.0.1')
-    run_fake( "source/connect_from_banned_ip")
+    run_fake('source/connect_from_banned_ip')
   end
 
-
   def test_bad_write
-    run_fake( "source/write_out_of_range" )
+    run_fake('source/write_out_of_range')
   end
 
-
   def test_disconnect_before_write_data_causes_error
-    run_fake( "source/close_after_write" )
+    run_fake('source/close_after_write')
   end
 
-
   def test_disconnect_before_write_reply_causes_error
     # Note that this is an odd case: writing the reply doesn't fail.
     # The test passes because the next attempt by flexnbd to read a
     # request returns EOF.
-    run_fake( "source/close_after_write_data" )
+    run_fake('source/close_after_write_data')
   end
 
-
-
   def test_straight_migration
     @env.nbd1.can_die(0)
-    run_fake( "source/successful_transfer" )
+    run_fake('source/successful_transfer')
   end
 
-
   private
+
   def run_fake(name)
     @env.run_fake(name, @env.ip, @env.port1)
     assert @env.fake_reports_success, "#{name} failed."
   end
 
   def status
-    stat, _ = @env.status1
+    stat, = @env.status1
     stat
   end
 
   def assert_no_control
-    assert !status['has_control'],  "Thought it had control"
+    assert !status['has_control'], 'Thought it had control'
   end
 
   def assert_control
     assert status['has_control'], "Didn't think it had control"
   end
-
 end # class TestDestErrorHandling
-

tests/acceptance/test_happy_path.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 require 'test/unit'
 require 'environment'
 require 'flexnbd/constants'
@@ -19,20 +17,18 @@ class TestHappyPath < Test::Unit::TestCase
     @env.cleanup
   end
 
-
   def test_read1
-    @env.writefile1("f"*64)
+    @env.writefile1('f' * 64)
     @env.serve1
 
     [0, 12, 63].each do |num|
-
       assert_equal(
         bin(@env.nbd1.read(num * @env.blocksize, @env.blocksize)),
         bin(@env.file1.read(num * @env.blocksize, @env.blocksize))
       )
     end
 
-    [124, 1200, 10028, 25488].each do |num|
+    [124, 1200, 10_028, 25_488].each do |num|
       assert_equal(bin(@env.nbd1.read(num, 4)), bin(@env.file1.read(num, 4)))
     end
   end
@@ -40,11 +36,11 @@ class TestHappyPath < Test::Unit::TestCase
   # Check that we're not
   #
   def test_writeread1
-    @env.writefile1("0"*64)
+    @env.writefile1('0' * 64)
     @env.serve1
 
     [0, 12, 63].each do |num|
-      data = "X"*@env.blocksize
+      data = 'X' * @env.blocksize
       @env.nbd1.write(num * @env.blocksize, data)
       assert_equal(data, @env.file1.read(num * @env.blocksize, data.size))
       assert_equal(data, @env.nbd1.read(num * @env.blocksize, data.size))
@@ -55,11 +51,11 @@ class TestHappyPath < Test::Unit::TestCase
   # up.
   #
   def test_writeread2
-    @env.writefile1("0"*1024)
+    @env.writefile1('0' * 1024)
     @env.serve1
 
     d0 = "\0" * @env.blocksize
-    d1 = "X"*@env.blocksize
+    d1 = 'X' * @env.blocksize
     (0..63).each do |num|
       @env.nbd1.write(num * @env.blocksize * 2, d1)
     end
@@ -68,20 +64,18 @@ class TestHappyPath < Test::Unit::TestCase
     end
   end
 
-
   def setup_to_mirror
-    @env.writefile1( "f"*4 )
+    @env.writefile1('f' * 4)
     @env.serve1
 
-    @env.writefile2( "0"*4 )
+    @env.writefile2('0' * 4)
     @env.listen2
   end
 
-
   def test_mirror
     @env.nbd1.can_die
     @env.nbd2.can_die(0)
-    setup_to_mirror()
+    setup_to_mirror
 
     stdout, stderr = @env.mirror12
 
@@ -89,16 +83,15 @@ class TestHappyPath < Test::Unit::TestCase
     @env.nbd2.join
 
     assert(File.file?(@env.filename1),
-           "The source file was incorrectly deleted")
+           'The source file was incorrectly deleted')
     assert_equal(@env.file1.read_original(0, @env.blocksize),
                  @env.file2.read(0, @env.blocksize))
   end
 
-
   def test_mirror_unlink
     @env.nbd1.can_die(0)
     @env.nbd2.can_die(0)
-    setup_to_mirror()
+    setup_to_mirror
 
     assert File.file?(@env.filename1)
 
@@ -106,45 +99,40 @@ class TestHappyPath < Test::Unit::TestCase
 
     assert_no_match(/unrecognized/, stderr)
 
-
+    Timeout.timeout(10) { @env.nbd1.join }
-    Timeout.timeout(10) do @env.nbd1.join end
 
     assert !File.file?(@env.filename1)
   end
 
-
-
   def test_write_to_high_block
     #
     # This test does not work on 32 bit platforms.
     #
-		skip("Not relevant on 32-bit platforms") if ( ["a"].pack("p").size < 8 )
+    skip('Not relevant on 32-bit platforms') if ['a'].pack('p').size < 8
 
     # Create a large file, then try to write to somewhere after the 2G boundary
-    @env.truncate1 "4G"
+    @env.truncate1 '4G'
     @env.serve1
 
-    @env.nbd1.write( 2**31+2**29, "12345678" )
+    @env.nbd1.write(2**31 + 2**29, '12345678')
     sleep(1)
-    assert_equal "12345678", @env.nbd1.read( 2**31+2**29, 8 )
+    assert_equal '12345678', @env.nbd1.read(2**31 + 2**29, 8)
   end
 
-
   def test_set_acl
     # Just check that we get sane feedback here
-    @env.writefile1( "f"*4 )
+    @env.writefile1('f' * 4)
     @env.serve1
 
-    _,stderr = @env.acl1("127.0.0.1")
+    _, stderr = @env.acl1('127.0.0.1')
     assert_no_match(/^(F|E):/, stderr)
   end
 
-
   def test_write_more_than_one_run
     one_mb = 2**20
     data = "\0" * 256 * one_mb
 
-    File.open(@env.filename1, "wb") do |f| f.write( "1" * 256 * one_mb ) end
+    File.open(@env.filename1, 'wb') { |f| f.write('1' * 256 * one_mb) }
 
     @env.serve1
     sleep 5
@@ -153,17 +141,15 @@ class TestHappyPath < Test::Unit::TestCase
     @env.nbd1.kill
 
     i = 0
-    File.open(@env.filename1, "rb") do |f|
+    File.open(@env.filename1, 'rb') do |f|
       while mb = f.read(one_mb)
         unless "\0" * one_mb == mb
-          msg = "Read non-zeros after offset %x:\n"%(i * one_mb)
+          msg = format("Read non-zeros after offset %x:\n", (i * one_mb))
           msg += `hexdump #{@env.filename1} | head -n5`
-          fail msg
+          raise msg
         end
         i += 1
       end
     end
   end
-
 end
-

tests/acceptance/test_prefetch_proxy_mode.rb

@@ -2,7 +2,6 @@ require 'test/unit'
 require 'environment'
 require 'proxy_tests'
 
-
 class TestPrefetchProxyMode < Test::Unit::TestCase
   include ProxyTests
 
@@ -10,7 +9,7 @@ class TestPrefetchProxyMode < Test::Unit::TestCase
     super
     @env = Environment.new
     @env.prefetch_proxy!
-    @env.writefile1( "f" * 16 )
+    @env.writefile1('f' * 16)
   end
 
   def teardown
@@ -18,5 +17,3 @@ class TestPrefetchProxyMode < Test::Unit::TestCase
     super
   end
 end
-
-

tests/acceptance/test_proxy_mode.rb

@@ -2,14 +2,13 @@ require 'test/unit'
 require 'environment'
 require 'proxy_tests'
 
-
 class TestProxyMode < Test::Unit::TestCase
   include ProxyTests
 
   def setup
     super
     @env = Environment.new
-    @env.writefile1( "f" * 16 )
+    @env.writefile1('f' * 16)
   end
 
   def teardown
@@ -17,4 +16,3 @@ class TestProxyMode < Test::Unit::TestCase
     super
   end
 end
-

tests/acceptance/test_serve_mode.rb

@@ -1,15 +1,15 @@
 require 'test/unit'
 require 'environment'
 require 'flexnbd/fake_source'
+require 'ld_preload'
 
 class TestServeMode < Test::Unit::TestCase
+  include LdPreload
 
   def setup
     super
     @b = "\xFF".b
     @env = Environment.new
-    @env.writefile1( "0" )
-    @env.serve1
   end
 
   def teardown
@@ -18,35 +18,44 @@ class TestServeMode < Test::Unit::TestCase
   end
 
   def connect_to_server
-    client = FlexNBD::FakeSource.new(@env.ip, @env.port1, "Connecting to server failed")
+    @env.writefile1('0')
+    @env.serve1
+    client = FlexNBD::FakeSource.new(@env.ip, @env.port1, 'Connecting to server failed')
     begin
       result = client.read_hello
-      assert_equal "NBDMAGIC", result[:magic]
+      assert_equal 'NBDMAGIC', result[:passwd]
+      assert_equal 0x00420281861253, result[:magic]
       assert_equal @env.file1.size, result[:size]
+      # See src/common/nbdtypes.h for the various flags. At the moment we
+      # support HAS_FLAGS (1), SEND_FLUSH (4), SEND_FUA (8)
+      assert_equal (1 | 4 | 8), result[:flags]
+      assert_equal "\x0" * 124, result[:reserved]
       yield client
     ensure
-      client.close rescue nil
+      begin
+        client.close
+      rescue StandardError
+        nil
+      end
     end
   end
 
   def test_bad_request_magic_receives_error_response
     connect_to_server do |client|
-
       # replace REQUEST_MAGIC with all 0s to make it look bad
-      client.send_request( 0, "myhandle", 0, 0, "\x00\x00\x00\x00" )
+      client.send_request(0, 'myhandle', 0, 0, "\x00\x00\x00\x00")
       rsp = client.read_response
 
       assert_equal FlexNBD::REPLY_MAGIC, rsp[:magic]
-      assert_equal "myhandle", rsp[:handle]
+      assert_equal 'myhandle', rsp[:handle]
       assert rsp[:error] != 0, "Server sent success reply back: #{rsp[:error]}"
 
       # The client should be disconnected now
-      assert client.disconnected?, "Server not disconnected"
+      assert client.disconnected?, 'Server not disconnected'
     end
   end
 
   def test_long_write_on_top_of_short_write_is_respected
-
     connect_to_server do |client|
       # Start with a file of all-zeroes.
       client.write(0, "\x00" * @env.file1.size)
@@ -68,15 +77,15 @@ class TestServeMode < Test::Unit::TestCase
     assert_equal @b * 2, @env.file1.read(0, 2)
   end
 
-
   def test_read_request_out_of_bounds_receives_error_response
     connect_to_server do |client|
       client.write_read_request(@env.file1.size, 4096)
       rsp = client.read_response
 
       assert_equal FlexNBD::REPLY_MAGIC, rsp[:magic]
-      assert_equal "myhandle", rsp[:handle]
+      assert_equal 'myhandle', rsp[:handle]
-      assert rsp[:error] != 0, "Server sent success reply back: #{rsp[:error]}"
+      # NBD protocol suggests ENOSPC (28) is returned
+      assert_equal 28, rsp[:error], 'Server sent incorrect response'
 
       # Ensure we're not disconnected by sending a request. We don't care about
       # whether the reply is good or not, here.
@@ -92,8 +101,9 @@ class TestServeMode < Test::Unit::TestCase
       rsp = client.read_response
 
       assert_equal FlexNBD::REPLY_MAGIC, rsp[:magic]
-      assert_equal "myhandle", rsp[:handle]
+      assert_equal 'myhandle', rsp[:handle]
-      assert rsp[:error] != 0, "Server sent success reply back: #{rsp[:error]}"
+      # NBD protocol suggests ENOSPC (28) is returned
+      assert_equal 28, rsp[:error], 'Server sent incorrect response'
 
       # Ensure we're not disconnected by sending a request. We don't care about
       # whether the reply is good or not, here.
@@ -101,10 +111,113 @@ class TestServeMode < Test::Unit::TestCase
       rsp = client.read_response
       assert_equal FlexNBD::REPLY_MAGIC, rsp[:magic]
     end
-
   end
 
+  def test_unknown_command_receives_error_response
+    connect_to_server do |client|
+      client.send_request(123)
+      rsp = client.read_response
 
+      assert_equal FlexNBD::REPLY_MAGIC, rsp[:magic]
+      assert_equal 'myhandle', rsp[:handle]
+      # NBD protocol suggests EINVAL (22) is returned
+      assert_equal 22, rsp[:error], 'Server sent incorrect response'
 
+      # Ensure we're not disconnected by sending a request. We don't care about
+      # whether the reply is good or not, here.
+      client.write(0, "\x00" * @env.file1.size)
+      rsp = client.read_response
+      assert_equal FlexNBD::REPLY_MAGIC, rsp[:magic]
+    end
   end
 
+  def test_flush_is_accepted
+    with_ld_preload('msync_logger') do
+      connect_to_server do |client|
+        client.flush
+        rsp = client.read_response
+        assert_equal FlexNBD::REPLY_MAGIC, rsp[:magic]
+        assert_equal 0, rsp[:error]
+      end
+      op = parse_ld_preload_logs('msync_logger')
+      assert_equal 1, op.count, 'Only one msync expected'
+      assert_equal @env.blocksize, op.first[2], 'msync length wrong'
+      assert_equal 6, op.first[3], 'msync called with incorrect flags'
+    end
+  end
+
+  def test_write_with_fua_is_accepted
+    with_ld_preload('msync_logger') do
+      page_size = Integer(`getconf PAGESIZE`)
+
+      @env.blocksize = page_size * 10
+      connect_to_server do |client|
+        # Write somewhere in the third page
+        pos = page_size * 3 + 100
+        client.write_with_fua(pos, "\x00" * 33)
+        rsp = client.read_response
+        assert_equal FlexNBD::REPLY_MAGIC, rsp[:magic]
+        assert_equal 0, rsp[:error]
+      end
+      op = parse_ld_preload_logs('msync_logger')
+
+      assert_equal 1, op.count, 'Only one msync expected'
+
+      # Should be 100 + 33, as we've started writing 100 bytes into a page, for
+      # 33 bytes
+      assert_equal 133, op.first[2], 'msync length wrong'
+      assert_equal 6, op.first[3], 'msync called with incorrect flags'
+    end
+  end
+
+  def test_odd_size_discs_are_truncated_to_nearest_512
+    # This should get rounded down to 1024
+    @env.blocksize = 1024 + 511
+    @env.writefile1('0')
+    @env.serve1
+    client = FlexNBD::FakeSource.new(@env.ip, @env.port1, 'Connecting to server failed')
+    begin
+      result = client.read_hello
+      assert_equal 'NBDMAGIC', result[:passwd]
+      assert_equal 0x00420281861253, result[:magic]
+      assert_equal 1024, result[:size]
+      client.close
+    end
+  end
+
+  def test_server_sets_tcpkeepalive
+    with_ld_preload('setsockopt_logger') do
+      connect_to_server(&:close)
+      op = read_ld_preload_log('setsockopt_logger')
+      assert_func_call(op,
+                       ['setsockopt', '\d+',
+                        Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, 1, 0],
+                       'TCP Keepalive not successfully set')
+      assert_func_call(op,
+                       ['setsockopt', '\d+',
+                        Socket::SOL_TCP, Socket::TCP_KEEPIDLE, 30, 0],
+                       'TCP Keepalive idle timeout not set to 30s')
+      assert_func_call(op,
+                       ['setsockopt', '\d+',
+                        Socket::SOL_TCP, Socket::TCP_KEEPINTVL, 10, 0],
+                       'TCP keepalive retry time not set to 10s')
+      assert_func_call(op,
+                       ['setsockopt', '\d+',
+                        Socket::SOL_TCP, Socket::TCP_KEEPCNT, 3, 0],
+                       'TCP keepalive count not set to 3')
+    end
+  end
+
+  def test_status_returns_correct_client_count
+    @env.writefile1('0')
+    @env.serve1
+    assert_equal('0', @env.status1['num_clients'])
+    client = FlexNBD::FakeSource.new(@env.ip, @env.port1, 'Connecting to server failed')
+    assert_equal('1', @env.status1['num_clients'])
+    client2 = FlexNBD::FakeSource.new(@env.ip, @env.port1, 'Connecting to server failed')
+    assert_equal('2', @env.status1['num_clients'])
+    client2.close
+    client.close
+    assert_equal('0', @env.status1['num_clients'])
+  end
+end

tests/acceptance/test_source_error_handling.rb

@@ -1,126 +1,105 @@
-# encoding: utf-8
-
 require 'test/unit'
 require 'environment'
 
-
 class TestSourceErrorHandling < Test::Unit::TestCase
-
   def setup
     @old_env = ENV['FLEXNBD_MS_REQUEST_LIMIT_SECS']
-    ENV['FLEXNBD_MS_REQUEST_LIMIT_SECS'] = "4.0"
+    ENV['FLEXNBD_MS_REQUEST_LIMIT_SECS'] = '4.0'
 
     @env = Environment.new
-    @env.writefile1( "f" * 4 )
+    @env.writefile1('f' * 4)
     @env.serve1
   end
 
-
   def teardown
     @env.nbd1.can_die(0)
     @env.cleanup
     ENV['FLEXNBD_MS_REQUEST_LIMIT_SECS'] = @old_env
   end
 
-
   def expect_term_during_migration
     @env.nbd1.can_die(1, 9)
   end
 
-
   def test_failure_to_connect_reported_in_mirror_cmd_response
     stdout, stderr = @env.mirror12_unchecked
     expect_term_during_migration
     assert_match(/failed to connect/, stderr)
   end
 
-
   def test_sigterm_after_hello_quits_with_status_of_1
     expect_term_during_migration
-    run_fake( "dest/sigterm_after_hello" )
+    run_fake('dest/sigterm_after_hello')
   end
 
-
   def test_destination_hangs_after_connect_reports_error_at_source
-    run_fake( "dest/hang_after_connect",
+    run_fake('dest/hang_after_connect',
-              :err => /Remote server failed to respond/ )
+             err: /Remote server failed to respond/)
   end
 
-
   def test_destination_rejects_connection_reports_error_at_source
-    run_fake( "dest/reject_acl",
+    run_fake('dest/reject_acl',
-              :err => /Mirror was rejected/ )
+             err: /Mirror was rejected/)
   end
 
-
   def test_wrong_size_causes_disconnect
-    run_fake( "dest/hello_wrong_size",
+    run_fake('dest/hello_wrong_size',
-              :err => /Remote size does not match local size/ )
+             err: /Remote size does not match local size/)
   end
 
-
   def test_wrong_magic_causes_disconnect
     expect_term_during_migration
-    run_fake( "dest/hello_wrong_magic",
+    run_fake('dest/hello_wrong_magic',
-              :err => /Mirror was rejected/ )
+             err: /Mirror was rejected/)
   end
 
-
   def test_disconnect_after_hello_causes_retry
     expect_term_during_migration
-    run_fake( "dest/close_after_hello",
+    run_fake('dest/close_after_hello',
-              :out => /Mirror started/ )
+             out: /Mirror started/)
   end
 
-
   def test_write_times_out_causes_retry
     expect_term_during_migration
-    run_fake( "dest/hang_after_write" )
+    run_fake('dest/hang_after_write')
   end
 
-
   def test_rejected_write_causes_retry
     expect_term_during_migration
-    run_fake( "dest/error_on_write" )
+    run_fake('dest/error_on_write')
   end
 
-
   def test_disconnect_before_write_reply_causes_retry
     expect_term_during_migration
-    run_fake( "dest/close_after_write" )
+    run_fake('dest/close_after_write')
   end
 
-
   def test_bad_write_reply_causes_retry
     expect_term_during_migration
-    run_fake( "dest/write_wrong_magic" )
+    run_fake('dest/write_wrong_magic')
   end
 
-
   def test_pre_entrust_disconnect_causes_retry
     expect_term_during_migration
-    run_fake( "dest/close_after_writes" )
+    run_fake('dest/close_after_writes')
   end
 
-
   def test_cancel_migration
-    run_fake( "dest/break_after_hello" )
+    run_fake('dest/break_after_hello')
   end
 
-
   private
+
   def run_fake(name, opts = {})
     @env.run_fake(name, @env.ip, @env.port2, @env.nbd1.ctrl)
     stdout, stderr = @env.mirror12_unchecked
     assert_success
     assert_match(opts[:err], stderr) if opts[:err]
     assert_match(opts[:out], stdout) if opts[:out]
-    return stdout, stderr
+    [stdout, stderr]
   end
 
   def assert_success(msg = nil)
-    assert @env.fake_reports_success, msg || "Fake failed"
+    assert @env.fake_reports_success, msg || 'Fake failed'
   end
-
-
 end # class TestSourceErrorHandling

tests/acceptance/test_write_during_migration.rb

@@ -9,114 +9,132 @@ require 'tmpdir'
 Thread.abort_on_exception = true
 
 class TestWriteDuringMigration < Test::Unit::TestCase
-
   def setup
-    @flexnbd = File.expand_path("../../build/flexnbd")
+    @flexnbd = File.expand_path('../../build/flexnbd')
-
-    raise "No binary!" unless File.executable?( @flexnbd )
 
+    raise 'No binary!' unless File.executable?(@flexnbd)
 
     @size = 20 * 1024 * 1024 # 20MB
-    @write_data = "foo!" * 2048 # 8K write
+    @write_data = 'foo!' * 2048 # 8K write
     @source_port = 9990
     @dest_port = 9991
-    @source_sock = "src.sock"
+    @source_sock = 'src.sock'
-    @dest_sock = "dst.sock"
+    @dest_sock = 'dst.sock'
-    @source_file = "src.file"
+    @source_file = 'src.file'
-    @dest_file = "dst.file"
+    @dest_file = 'dst.file'
   end
 
-
   def teardown
     [@dst_proc, @src_proc].each do |pid|
-      if pid
+      next unless pid
-        Process.kill( "KILL", pid ) rescue nil
+      begin
+        Process.kill('KILL', pid)
+      rescue StandardError
+        nil
       end
     end
   end
 
   def debug_arg
-    ENV['DEBUG'] ? "--verbose" : ""
+    ENV['DEBUG'] ? '--verbose' : ''
   end
 
-
   def launch_servers
-    @dst_proc = fork() {
+    @dst_proc = fork do
       cmd = "#{@flexnbd} listen -l 127.0.0.1 -p #{@dest_port} -f #{@dest_file} -s #{@dest_sock} #{debug_arg}"
       exec cmd
-    }
+    end
 
-    @src_proc = fork() {
+    @src_proc = fork do
       cmd = "#{@flexnbd} serve -l 127.0.0.1 -p #{@source_port} -f #{@source_file} -s #{@source_sock} #{debug_arg}"
       exec cmd
-    }
+    end
     begin
       awaiting = nil
       Timeout.timeout(10) do
         awaiting = :source
-        sleep 0.1 while !File.exists?( @source_sock )
+        sleep 0.1 until File.exist?(@source_sock)
         awaiting = :dest
-        sleep 0.1 while !File.exists?( @dest_sock )
+        sleep 0.1 until File.exist?(@dest_sock)
       end
     rescue Timeout::Error
       case awaiting
       when :source
-        fail "Couldn't get a source socket."
+        raise "Couldn't get a source socket."
       when :dest
-        fail "Couldn't get a destination socket."
+        raise "Couldn't get a destination socket."
       else
-        fail "Something went wrong I don't understand."
+        raise "Something went wrong I don't understand."
       end
     end
   end
 
-
+  def make_files
-  def make_files()
     FileUtils.touch(@source_file)
     File.truncate(@source_file, @size)
     FileUtils.touch(@dest_file)
     File.truncate(@dest_file, @size)
 
-    File.open(@source_file, "wb"){|f| f.write "a"*@size }
+    File.open(@source_file, 'wb') { |f| f.write 'a' * @size }
   end
 
-
   def start_mirror
-    UNIXSocket.open(@source_sock) {|sock|
+    UNIXSocket.open(@source_sock) do |sock|
-      sock.write(["mirror", "127.0.0.1", @dest_port.to_s, "exit"].join("\x0A") + "\x0A\x0A")
+      sock.write(['mirror', '127.0.0.1', @dest_port.to_s, 'exit'].join("\x0A") + "\x0A\x0A")
       sock.flush
-      rsp = sock.readline
+      sock.readline
-    }
+    end
   end
 
+  def stop_mirror
+    UNIXSocket.open(@source_sock) do |sock|
+      sock.write("break\x0A\x0A")
+      sock.flush
+      sock.readline
+    end
+  end
 
-  def wait_for_quit()
+  def wait_for_quit
     Timeout.timeout(10) do
-      start_time =  Time.now
+      Process.waitpid2(@dst_proc)
-      dst_result = Process::waitpid2(@dst_proc)
+      Process.waitpid2(@src_proc)
-      src_result = Process::waitpid2(@src_proc)
     end
   end
 
   def source_writer
-    client = FlexNBD::FakeSource.new( "127.0.0.1", @source_port, "Timed out connecting" )
+    client = FlexNBD::FakeSource.new('127.0.0.1', @source_port, 'Timed out connecting')
     offsets = Range.new(0, (@size - @write_data.size) / 4096).to_a
     loop do
       begin
         client.write(offsets[rand(offsets.size)] * 4096, @write_data)
-      rescue => err
+      rescue StandardError
         # We expect a broken write at some point, so ignore it
         break
       end
     end
   end
 
+  def bombard_with_status
+    loop do
+      begin
+        UNIXSocket.open(@source_sock) do |sock|
+          sock.write("status\x0A\x0A")
+          sock.flush
+          sock.readline
+        end
+      rescue StandardError
+        # If the socket disappears, that's OK.
+        break
+      end
+    end
+  end
+
   def assert_both_sides_identical
     # puts `md5sum #{@source_file} #{@dest_file}`
 
     # Ensure each block matches
-    File.open(@source_file, "r") do |source|
+    File.open(@source_file, 'r') do |source|
-      File.open(@dest_file, "r") do |dest|
+      File.open(@dest_file, 'r') do |dest|
         0.upto(@size / 4096) do |block_num|
           s_data = source.read(4096)
           d_data = dest.read(4096)
@@ -131,16 +149,16 @@ class TestWriteDuringMigration < Test::Unit::TestCase
   end
 
   def test_write_during_migration
-    Dir.mktmpdir() do |tmpdir|
+    Dir.mktmpdir do |tmpdir|
       Dir.chdir(tmpdir) do
-        make_files()
+        make_files
 
-        launch_servers()
+        launch_servers
 
         src_writer = Thread.new { source_writer }
 
-        start_mirror()
+        start_mirror
-        wait_for_quit()
+        wait_for_quit
         src_writer.join
         assert_both_sides_identical
       end
@@ -148,24 +166,64 @@ class TestWriteDuringMigration < Test::Unit::TestCase
   end
 
   def test_many_clients_during_migration
-    Dir.mktmpdir() do |tmpdir|
+    Dir.mktmpdir do |tmpdir|
       Dir.chdir(tmpdir) do
-        make_files()
+        make_files
 
-        launch_servers()
+        launch_servers
 
         src_writers_1 = (1..5).collect { Thread.new { source_writer } }
 
-        start_mirror()
+        start_mirror
 
         src_writers_2 = (1..5).collect { Thread.new { source_writer } }
 
-        wait_for_quit()
+        wait_for_quit
-        ( src_writers_1 + src_writers_2 ).each {|t| t.join }
+        (src_writers_1 + src_writers_2).each(&:join)
         assert_both_sides_identical
       end
-    end  end
+    end
-
-
   end
 
+  def test_status_call_after_cleanup
+    Dir.mktmpdir do |tmpdir|
+      Dir.chdir(tmpdir) do
+        make_files
+
+        launch_servers
+
+        status_poker = Thread.new { bombard_with_status }
+
+        start_mirror
+
+        wait_for_quit
+        status_poker.join
+        assert_both_sides_identical
+      end
+    end
+  end
+
+  def test_mirroring_can_be_restarted
+    @size = 100 * 1024 * 1024 # 100MB
+    Dir.mktmpdir do |tmpdir|
+      Dir.chdir(tmpdir) do
+        make_files
+
+        launch_servers
+
+        # This is a bit racy.  It needs to be slow enough that the migration
+        # isn't finished before the stop runs, and slow enough so that we can
+        # stop/start a few times.
+        3.times do
+          start_mirror
+          sleep 0.1
+          stop_mirror
+          sleep 0.1
+        end
+        start_mirror
+
+        wait_for_quit
+      end
+    end
+  end
+end

tests/unit/check_acl.c

@@ -13,7 +13,6 @@ START_TEST( test_null_acl )
 }
 END_TEST
 
-
 START_TEST(test_parses_single_line)
 {
     char *lines[] = { "127.0.0.1" };
@@ -37,9 +36,11 @@ START_TEST( test_parses_multiple_lines )
 
     struct ip_and_mask *entry;
     entry = &(*acl->entries)[0];
-	fail_unless(entry->ip.family == e0.family, "entry 0 has wrong family!");
+    fail_unless(entry->ip.family == e0.family,
+		"entry 0 has wrong family!");
     entry = &(*acl->entries)[1];
-	fail_unless(entry->ip.family == e1.family, "entry 1 has wrong family!");
+    fail_unless(entry->ip.family == e1.family,
+		"entry 1 has wrong family!");
 }
 END_TEST
 
@@ -52,7 +53,6 @@ START_TEST( test_destroy_doesnt_crash )
 }
 END_TEST
 
-
 START_TEST(test_includes_single_address)
 {
     char *lines[] = { "127.0.0.1" };
@@ -109,12 +109,13 @@ START_TEST( test_includes_single_address_when_multiple_entries_exist )
     parse_ip_to_sockaddr(&e0.generic, "127.0.0.1");
     parse_ip_to_sockaddr(&e1.generic, "::1");
 
-	fail_unless( acl_includes( acl, &e0 ), "Included address 0 wasn't covered" );
+    fail_unless(acl_includes(acl, &e0),
-	fail_unless( acl_includes( acl, &e1 ), "Included address 1 wasn't covered" );
+		"Included address 0 wasn't covered");
+    fail_unless(acl_includes(acl, &e1),
+		"Included address 1 wasn't covered");
 }
 END_TEST
 
-
 START_TEST(test_doesnt_include_other_address)
 {
     char *lines[] = { "127.0.0.1" };
@@ -163,7 +164,6 @@ START_TEST( test_default_deny_rejects )
 }
 END_TEST
 
-
 START_TEST(test_default_accept_rejects)
 {
     struct acl *acl = acl_create(0, NULL, 0);
@@ -175,7 +175,6 @@ START_TEST( test_default_accept_rejects )
 }
 END_TEST
 
-
 Suite * acl_suite(void)
 {
     Suite *s = suite_create("acl");
@@ -189,14 +188,19 @@ Suite* acl_suite(void)
     tcase_add_test(tc_includes, test_parses_multiple_lines);
 
     tcase_add_test(tc_includes, test_includes_single_address);
-	tcase_add_test(tc_includes, test_includes_single_address_when_netmask_specified_ipv4);
+    tcase_add_test(tc_includes,
-	tcase_add_test(tc_includes, test_includes_single_address_when_netmask_specified_ipv6);
+		   test_includes_single_address_when_netmask_specified_ipv4);
+    tcase_add_test(tc_includes,
+		   test_includes_single_address_when_netmask_specified_ipv6);
 
-	tcase_add_test(tc_includes, test_includes_single_address_when_multiple_entries_exist);
+    tcase_add_test(tc_includes,
+		   test_includes_single_address_when_multiple_entries_exist);
 
     tcase_add_test(tc_includes, test_doesnt_include_other_address);
-	tcase_add_test(tc_includes, test_doesnt_include_other_address_when_netmask_specified);
+    tcase_add_test(tc_includes,
-	tcase_add_test(tc_includes, test_doesnt_include_other_address_when_multiple_entries_exist);
+		   test_doesnt_include_other_address_when_netmask_specified);
+    tcase_add_test(tc_includes,
+		   test_doesnt_include_other_address_when_multiple_entries_exist);
 
     tcase_add_test(tc_includes, test_default_deny_rejects);
     tcase_add_test(tc_includes, test_default_accept_rejects);
@@ -226,4 +230,3 @@ int main(void)
     srunner_free(sr);
     return (number_failed == 0) ? 0 : 1;
 }
-

tests/unit/check_bitset.c

@@ -66,18 +66,18 @@ START_TEST(test_bit_ranges)
 
     for (i = 0; i < 64; i++) {
 	bit_set_range(buffer, i * 64, i);
-		fail_unless(
+	fail_unless(longs[i] == (1ULL << i) - 1,
-			longs[i] == (1ULL<<i)-1,
 		    "longs[%ld] = %lx SHOULD BE %lx",
-			i, longs[i], (1ULL<<i)-1
+		    i, longs[i], (1ULL << i) - 1);
-		);
 
-		fail_unless(longs[i+1] == 0, "bit_set_range overshot at i=%d", i);
+	fail_unless(longs[i + 1] == 0, "bit_set_range overshot at i=%d",
+		    i);
     }
 
     for (i = 0; i < 64; i++) {
 	bit_clear_range(buffer, i * 64, i);
-		fail_unless(longs[i] == 0, "bit_clear_range didn't work at i=%d", i);
+	fail_unless(longs[i] == 0, "bit_clear_range didn't work at i=%d",
+		    i);
     }
 }
 END_TEST
@@ -86,7 +86,8 @@ START_TEST(test_bit_runs)
 {
     bitfield_word_t buffer[BIT_WORDS_FOR_SIZE(256)];
     int i, ptr = 0, runs[] = {
-		56,97,22,12,83,1,45,80,85,51,64,40,63,67,75,64,94,81,79,62
+	56, 97, 22, 12, 83, 1, 45, 80, 85, 51, 64, 40, 63, 67, 75, 64, 94,
+	    81, 79, 62
     };
 
     memset(buffer, 0, 256);
@@ -101,11 +102,8 @@ START_TEST(test_bit_runs)
 
     for (i = 0; i < 20; i += 1) {
 	int run = bit_run_count(buffer, ptr, 2048 - ptr, NULL);
-		fail_unless(
+	fail_unless(run == runs[i],
-			run == runs[i],
+		    "run %d should have been %d, was %d", i, runs[i], run);
-			"run %d should have been %d, was %d",
-			i, runs[i], run
-		);
 	ptr += runs[i];
     }
 }
@@ -145,7 +143,6 @@ START_TEST(test_bitset)
 }
 END_TEST
 
-
 START_TEST(test_bitset_set)
 {
     struct bitset *map;
@@ -176,7 +173,6 @@ START_TEST( test_bitset_set )
 }
 END_TEST
 
-
 START_TEST(test_bitset_clear)
 {
     struct bitset *map;
@@ -443,10 +439,14 @@ START_TEST(test_bitset_stream_queued_bytes)
     bitset_clear_range(map, 0, 2);
     bitset_disable_stream(map);
 
-	ck_assert_int_eq( 64, bitset_stream_queued_bytes( map, BITSET_STREAM_ON ) );
+    ck_assert_int_eq(64,
-	ck_assert_int_eq( 80, bitset_stream_queued_bytes( map, BITSET_STREAM_SET ) );
+		     bitset_stream_queued_bytes(map, BITSET_STREAM_ON));
-	ck_assert_int_eq( 82, bitset_stream_queued_bytes( map, BITSET_STREAM_UNSET ) );
+    ck_assert_int_eq(80,
-	ck_assert_int_eq( 64, bitset_stream_queued_bytes( map, BITSET_STREAM_OFF ) );
+		     bitset_stream_queued_bytes(map, BITSET_STREAM_SET));
+    ck_assert_int_eq(82,
+		     bitset_stream_queued_bytes(map, BITSET_STREAM_UNSET));
+    ck_assert_int_eq(64,
+		     bitset_stream_queued_bytes(map, BITSET_STREAM_OFF));
     bitset_free(map);
 }
 END_TEST
@@ -471,7 +471,8 @@ Suite* bitset_suite(void)
     tcase_add_test(tc_bitset, test_bitset_set_range);
     tcase_add_test(tc_bitset, test_bitset_clear_range);
     tcase_add_test(tc_bitset, test_bitset_set_range_doesnt_push_to_stream);
-	tcase_add_test(tc_bitset, test_bitset_clear_range_doesnt_push_to_stream);
+    tcase_add_test(tc_bitset,
+		   test_bitset_clear_range_doesnt_push_to_stream);
     suite_add_tcase(s, tc_bitset);
 
 
@@ -497,4 +498,3 @@ int main(void)
     srunner_free(sr);
     return (number_failed == 0) ? 0 : 1;
 }
-

tests/unit/check_client.c

@@ -10,6 +10,7 @@
 #include <unistd.h>
 
 struct server fake_server = { 0 };
+
 #define FAKE_SERVER &fake_server
 #define FAKE_SOCKET (42)
 
@@ -23,7 +24,6 @@ START_TEST( test_assigns_socket )
 }
 END_TEST
 
-
 START_TEST(test_assigns_server)
 {
     struct client *c;
@@ -37,7 +37,6 @@ START_TEST( test_assigns_server )
 }
 END_TEST
 
-
 START_TEST(test_opens_stop_signal)
 {
     struct client *c = client_create(FAKE_SERVER, FAKE_SOCKET);
@@ -50,7 +49,6 @@ START_TEST( test_opens_stop_signal )
 }
 END_TEST
 
-
 int fd_is_closed(int);
 
 START_TEST(test_closes_stop_signal)
@@ -66,7 +64,6 @@ START_TEST( test_closes_stop_signal )
 }
 END_TEST
 
-
 START_TEST(test_read_request_quits_on_stop_signal)
 {
     int fds[2];
@@ -84,7 +81,6 @@ START_TEST( test_read_request_quits_on_stop_signal )
 }
 END_TEST
 
-
 Suite * client_suite(void)
 {
     Suite *s = suite_create("client");
@@ -119,4 +115,3 @@ int main(void)
     srunner_free(sr);
     return (number_failed == 0) ? 0 : 1;
 }
-

tests/unit/check_control.c

@@ -15,7 +15,6 @@ START_TEST( test_assigns_sock_name )
 }
 END_TEST
 
-
 Suite * control_suite(void)
 {
     Suite *s = suite_create("control");
@@ -39,4 +38,3 @@ int main(void)
     srunner_free(sr);
     return (number_failed == 0) ? 0 : 1;
 }
-

tests/unit/check_flexnbd.c

@@ -5,8 +5,7 @@
 
 START_TEST(test_listening_assigns_sock)
 {
-	struct flexnbd * flexnbd = flexnbd_create_listening(
+    struct flexnbd *flexnbd = flexnbd_create_listening("127.0.0.1",
-			"127.0.0.1",
 						       "4777",
 						       "fakefile",
 						       "fakesock",
@@ -17,7 +16,6 @@ START_TEST( test_listening_assigns_sock )
 }
 END_TEST
 
-
 Suite * flexnbd_suite(void)
 {
     Suite *s = suite_create("flexnbd");
@@ -41,4 +39,3 @@ int main(void)
     srunner_free(sr);
     return (number_failed == 0) ? 0 : 1;
 }
-

tests/unit/check_flexthread.c

@@ -12,22 +12,23 @@ START_TEST( test_mutex_create )
 }
 END_TEST
 
-
 START_TEST(test_mutex_lock)
 {
     struct flexthread_mutex *ftm = flexthread_mutex_create();
 
-	fail_if( flexthread_mutex_held( ftm ), "Flexthread_mutex is held before lock" );
+    fail_if(flexthread_mutex_held(ftm),
+	    "Flexthread_mutex is held before lock");
     flexthread_mutex_lock(ftm);
-	fail_unless( flexthread_mutex_held( ftm ), "Flexthread_mutex is not held inside lock" );
+    fail_unless(flexthread_mutex_held(ftm),
+		"Flexthread_mutex is not held inside lock");
     flexthread_mutex_unlock(ftm);
-	fail_if( flexthread_mutex_held( ftm ), "Flexthread_mutex is held after unlock" );
+    fail_if(flexthread_mutex_held(ftm),
+	    "Flexthread_mutex is held after unlock");
 
     flexthread_mutex_destroy(ftm);
 }
 END_TEST
 
-
 Suite * flexthread_suite(void)
 {
     Suite *s = suite_create("flexthread");
@@ -59,4 +60,3 @@ int main(void)
     srunner_free(sr);
     return (number_failed == 0) ? 0 : 1;
 }
-

tests/unit/check_ioutil.c

@@ -17,7 +17,6 @@ START_TEST( test_read_until_newline_returns_line_length_plus_null )
 }
 END_TEST
 
-
 START_TEST(test_read_until_newline_inserts_null)
 {
     int fds[2];
@@ -34,7 +33,6 @@ START_TEST( test_read_until_newline_inserts_null )
 }
 END_TEST
 
-
 START_TEST(test_read_empty_line_inserts_null)
 {
     int fds[2];
@@ -50,7 +48,6 @@ START_TEST( test_read_empty_line_inserts_null )
 }
 END_TEST
 
-
 START_TEST(test_read_eof_returns_err)
 {
     int fds[2];
@@ -65,7 +62,6 @@ START_TEST( test_read_eof_returns_err )
 }
 END_TEST
 
-
 START_TEST(test_read_eof_fills_line)
 {
     int fds[2];
@@ -82,7 +78,6 @@ START_TEST( test_read_eof_fills_line )
 }
 END_TEST
 
-
 START_TEST(test_read_lines_until_blankline)
 {
     char **lines = NULL;
@@ -98,21 +93,25 @@ START_TEST( test_read_lines_until_blankline )
 }
 END_TEST
 
-
 Suite * ioutil_suite(void)
 {
     Suite *s = suite_create("ioutil");
 
     TCase *tc_read_until_newline = tcase_create("read_until_newline");
-	TCase *tc_read_lines_until_blankline = tcase_create("read_lines_until_blankline");
+    TCase *tc_read_lines_until_blankline =
+	tcase_create("read_lines_until_blankline");
 
-	tcase_add_test(tc_read_until_newline, test_read_until_newline_returns_line_length_plus_null);
+    tcase_add_test(tc_read_until_newline,
-	tcase_add_test(tc_read_until_newline, test_read_until_newline_inserts_null);
+		   test_read_until_newline_returns_line_length_plus_null);
-	tcase_add_test(tc_read_until_newline, test_read_empty_line_inserts_null);
+    tcase_add_test(tc_read_until_newline,
+		   test_read_until_newline_inserts_null);
+    tcase_add_test(tc_read_until_newline,
+		   test_read_empty_line_inserts_null);
     tcase_add_test(tc_read_until_newline, test_read_eof_returns_err);
     tcase_add_test(tc_read_until_newline, test_read_eof_fills_line);
 
-	tcase_add_test(tc_read_lines_until_blankline, test_read_lines_until_blankline );
+    tcase_add_test(tc_read_lines_until_blankline,
+		   test_read_lines_until_blankline);
 
     suite_add_tcase(s, tc_read_until_newline);
     suite_add_tcase(s, tc_read_lines_until_blankline);
@@ -131,4 +130,3 @@ int main(void)
     srunner_free(sr);
     return (number_failed == 0) ? 0 : 1;
 }
-

tests/unit/check_mbox.c

@@ -14,12 +14,11 @@ START_TEST( test_allocs_cvar )
     memset(&cond_zero, 'X', sizeof(cond_zero));
     fail_if(memcmp(&cond_zero, &mbox->filled_cond, sizeof(cond_zero)) == 0,
 	    "Condition variable not allocated");
-	fail_if( memcmp( &cond_zero, &mbox->emptied_cond, sizeof( cond_zero ) ) == 0 ,
+    fail_if(memcmp(&cond_zero, &mbox->emptied_cond, sizeof(cond_zero)) ==
-			"Condition variable not allocated" );
+	    0, "Condition variable not allocated");
 }
 END_TEST
 
-
 START_TEST(test_post_stores_value)
 {
     struct mbox *mbox = mbox_create();
@@ -32,7 +31,6 @@ START_TEST( test_post_stores_value )
 }
 END_TEST
 
-
 void *mbox_receive_runner(void *mbox_uncast)
 {
     struct mbox *mbox = (struct mbox *) mbox_uncast;
@@ -58,14 +56,12 @@ START_TEST( test_receive_blocks_until_post )
     mbox_post(mbox, deadbeef);
     fail_unless(0 == pthread_join(receiver, &retval),
 		"Failed to join the receiver thread");
-	fail_unless( retval == deadbeef,
+    fail_unless(retval == deadbeef, "Return value was wrong");
-			"Return value was wrong" );
 
 
 }
 END_TEST
 
-
 Suite * mbox_suite(void)
 {
     Suite *s = suite_create("mbox");
@@ -101,4 +97,3 @@ int main(void)
     srunner_free(sr);
     return (number_failed == 0) ? 0 : 1;
 }
-

tests/unit/check_nbdtypes.c

@@ -13,12 +13,13 @@ START_TEST(test_init_passwd)
     memset(init_raw.passwd, 0, 8);
     nbd_h2r_init(&init, &init_raw);
 
-	fail_unless( memcmp( init.passwd, INIT_PASSWD, 8 ) == 0, "The password was not copied." );
+    fail_unless(memcmp(init.passwd, INIT_PASSWD, 8) == 0,
-	fail_unless( memcmp( init_raw.passwd, INIT_PASSWD, 8 ) == 0, "The password was not copied back." );
+		"The password was not copied.");
+    fail_unless(memcmp(init_raw.passwd, INIT_PASSWD, 8) == 0,
+		"The password was not copied back.");
 }
 END_TEST
 
-
 START_TEST(test_init_magic)
 {
     struct nbd_init_raw init_raw;
@@ -30,11 +31,11 @@ START_TEST(test_init_magic)
 
     init.magic = 67890;
     nbd_h2r_init(&init, &init_raw);
-	fail_unless( htobe64( 67890 ) == init_raw.magic, "Magic was not converted back." );
+    fail_unless(htobe64(67890) == init_raw.magic,
+		"Magic was not converted back.");
 }
 END_TEST
 
-
 START_TEST(test_init_size)
 {
     struct nbd_init_raw init_raw;
@@ -46,11 +47,11 @@ START_TEST(test_init_size)
 
     init.size = 67890;
     nbd_h2r_init(&init, &init_raw);
-	fail_unless( htobe64( 67890 ) == init_raw.size, "Size was not converted back." );
+    fail_unless(htobe64(67890) == init_raw.size,
+		"Size was not converted back.");
 }
 END_TEST
 
-
 START_TEST(test_request_magic)
 {
     struct nbd_request_raw request_raw;
@@ -58,11 +59,13 @@ START_TEST(test_request_magic )
 
     request_raw.magic = 12345;
     nbd_r2h_request(&request_raw, &request);
-	fail_unless( be32toh( 12345 ) == request.magic, "Magic was not converted." );
+    fail_unless(be32toh(12345) == request.magic,
+		"Magic was not converted.");
 
     request.magic = 67890;
     nbd_h2r_request(&request, &request_raw);
-	fail_unless( htobe32( 67890 ) == request_raw.magic, "Magic was not converted back." );
+    fail_unless(htobe32(67890) == request_raw.magic,
+		"Magic was not converted back.");
 }
 END_TEST
 
@@ -71,17 +74,33 @@ START_TEST(test_request_type )
     struct nbd_request_raw request_raw;
     struct nbd_request request;
 
-	request_raw.type = 12345;
+    request_raw.type = 123;
     nbd_r2h_request(&request_raw, &request);
-	fail_unless( be32toh( 12345 ) == request.type, "Type was not converted." );
+    fail_unless(be16toh(123) == request.type, "Type was not converted.");
 
-	request.type = 67890;
+    request.type = 234;
     nbd_h2r_request(&request, &request_raw);
-	fail_unless( htobe32( 67890 ) == request_raw.type, "Type was not converted back." );
+    fail_unless(htobe16(234) == request_raw.type,
+		"Type was not converted back.");
 }
 END_TEST
 
+START_TEST(test_request_flags)
+{
+    struct nbd_request_raw request_raw;
+    struct nbd_request request;
 
+    request_raw.flags = 123;
+    nbd_r2h_request(&request_raw, &request);
+    fail_unless(be16toh(123) == request.flags,
+		"Flags were not converted.");
+
+    request.flags = 234;
+    nbd_h2r_request(&request, &request_raw);
+    fail_unless(htobe16(234) == request_raw.flags,
+		"Flags were not converted back.");
+}
+END_TEST
 
 START_TEST(test_request_handle)
 {
@@ -94,13 +113,13 @@ START_TEST(test_request_handle)
     request_raw.handle.w = 0;
     nbd_h2r_request(&request, &request_raw);
 
-	fail_unless( memcmp( request.handle.b, "MYHANDLE", 8 ) == 0, "The handle was not copied." );
+    fail_unless(memcmp(request.handle.b, "MYHANDLE", 8) == 0,
-	fail_unless( memcmp( request_raw.handle.b, "MYHANDLE", 8 ) == 0, "The handle was not copied back." );
+		"The handle was not copied.");
+    fail_unless(memcmp(request_raw.handle.b, "MYHANDLE", 8) == 0,
+		"The handle was not copied back.");
 }
 END_TEST
 
-
-
 START_TEST(test_request_from)
 {
     struct nbd_request_raw request_raw;
@@ -112,12 +131,11 @@ START_TEST(test_request_from )
 
     request.from = 67890;
     nbd_h2r_request(&request, &request_raw);
-	fail_unless( htobe64( 67890 ) == request_raw.from, "From was not converted back." );
+    fail_unless(htobe64(67890) == request_raw.from,
+		"From was not converted back.");
 }
 END_TEST
 
-
-
 START_TEST(test_request_len)
 {
     struct nbd_request_raw request_raw;
@@ -129,11 +147,11 @@ START_TEST(test_request_len )
 
     request.len = 67890;
     nbd_h2r_request(&request, &request_raw);
-	fail_unless( htobe32( 67890 ) == request_raw.len, "Type was not converted back." );
+    fail_unless(htobe32(67890) == request_raw.len,
+		"Type was not converted back.");
 }
 END_TEST
 
-
 START_TEST(test_reply_magic)
 {
     struct nbd_reply_raw reply_raw;
@@ -145,11 +163,11 @@ START_TEST(test_reply_magic )
 
     reply.magic = 67890;
     nbd_h2r_reply(&reply, &reply_raw);
-	fail_unless( htobe32( 67890 ) == reply_raw.magic, "Magic was not converted back." );
+    fail_unless(htobe32(67890) == reply_raw.magic,
+		"Magic was not converted back.");
 }
 END_TEST
 
-
 START_TEST(test_reply_error)
 {
     struct nbd_reply_raw reply_raw;
@@ -161,7 +179,8 @@ START_TEST(test_reply_error )
 
     reply.error = 67890;
     nbd_h2r_reply(&reply, &reply_raw);
-	fail_unless( htobe32( 67890 ) == reply_raw.error, "Error was not converted back." );
+    fail_unless(htobe32(67890) == reply_raw.error,
+		"Error was not converted back.");
 }
 END_TEST
 
@@ -176,12 +195,13 @@ START_TEST(test_reply_handle)
     reply_raw.handle.w = 0;
     nbd_h2r_reply(&reply, &reply_raw);
 
-	fail_unless( memcmp( reply.handle.b, "MYHANDLE", 8 ) == 0, "The handle was not copied." );
+    fail_unless(memcmp(reply.handle.b, "MYHANDLE", 8) == 0,
-	fail_unless( memcmp( reply_raw.handle.b, "MYHANDLE", 8 ) == 0, "The handle was not copied back." );
+		"The handle was not copied.");
+    fail_unless(memcmp(reply_raw.handle.b, "MYHANDLE", 8) == 0,
+		"The handle was not copied back.");
 }
 END_TEST
 
-
 START_TEST(test_convert_from)
 {
     /* Check that we can correctly pull numbers out of an
@@ -213,6 +233,7 @@ Suite *nbdtypes_suite(void)
     tcase_add_test(tc_init, test_init_size);
     tcase_add_test(tc_request, test_request_magic);
     tcase_add_test(tc_request, test_request_type);
+    tcase_add_test(tc_request, test_request_flags);
     tcase_add_test(tc_request, test_request_handle);
     tcase_add_test(tc_request, test_request_from);
     tcase_add_test(tc_request, test_request_len);
@@ -240,4 +261,3 @@ int main(void)
     srunner_free(sr);
     return (number_failed == 0) ? 0 : 1;
 }
-

tests/unit/check_parse.c

@@ -13,8 +13,6 @@ START_TEST( test_can_parse_ip_address_twice )
 }
 END_TEST
 
-
-
 Suite * parse_suite(void)
 {
     Suite *s = suite_create("parse");
@@ -44,4 +42,3 @@ int main(void)
     srunner_free(sr);
     return (number_failed == 0) ? 0 : 1;
 }
-

tests/unit/check_readwrite.c

@@ -54,8 +54,7 @@ void * responder( void *respond_uncast )
 	nbd_r2h_request(&request_raw, &resp->received);
 	if (resp->do_fail) {
 	    fd_write_reply(sock_fd, wrong_handle, 0);
-		}
+	} else {
-		else {
 	    fd_write_reply(sock_fd, resp->received.handle.w, 0);
 	}
 	write(sock_fd, "12345678", 8);
@@ -66,17 +65,20 @@ void * responder( void *respond_uncast )
 
 struct respond *respond_create(int do_fail)
 {
-	struct respond * respond = (struct respond *)calloc( 1, sizeof( struct respond ) );
+    struct respond *respond =
+	(struct respond *) calloc(1, sizeof(struct respond));
     socketpair(PF_UNIX, SOCK_STREAM, 0, respond->sock_fds);
     respond->do_fail = do_fail;
 
     pthread_attr_init(&respond->thread_attr);
-	pthread_create( &respond->thread_id, &respond->thread_attr, responder, respond );
+    pthread_create(&respond->thread_id, &respond->thread_attr, responder,
+		   respond);
 
     return respond;
 }
 
-void respond_destroy( struct respond * respond ){
+void respond_destroy(struct respond *respond)
+{
     NULLCHECK(respond);
 
     pthread_join(respond->thread_id, NULL);
@@ -121,7 +123,6 @@ START_TEST( test_rejects_mismatched_handle )
 }
 END_TEST
 
-
 START_TEST(test_accepts_matched_handle)
 {
     struct respond *respond = respond_create(0);
@@ -135,7 +136,6 @@ START_TEST( test_accepts_matched_handle )
 }
 END_TEST
 
-
 START_TEST(test_disconnect_doesnt_read_reply)
 {
     struct respond *respond = respond_create(1);
@@ -146,7 +146,6 @@ START_TEST( test_disconnect_doesnt_read_reply )
 }
 END_TEST
 
-
 Suite * readwrite_suite(void)
 {
     Suite *s = suite_create("readwrite");
@@ -162,7 +161,8 @@ Suite* readwrite_suite(void)
      * because we want to know that the sender won't even try to
      * read the response.
      */
-	tcase_add_exit_test( tc_disconnect, test_disconnect_doesnt_read_reply,0 );
+    tcase_add_exit_test(tc_disconnect, test_disconnect_doesnt_read_reply,
+			0);
 
     suite_add_tcase(s, tc_transfer);
     suite_add_tcase(s, tc_disconnect);
@@ -190,4 +190,3 @@ int main(void)
     srunner_free(sr);
     return (number_failed == 0) ? 0 : 1;
 }
-

tests/unit/check_self_pipe.c

@@ -24,7 +24,6 @@ START_TEST( test_opens_pipe )
 }
 END_TEST
 
-
 void *signal_thread(void *thing)
 {
     struct self_pipe *sig = (struct self_pipe *) thing;
@@ -63,14 +62,14 @@ START_TEST( test_signals )
     }
     self_pipe_signal_clear(sig);
 
-	fail_unless( self_pipe_fd_isset( sig, &fds ), "Signalled pipe was not FD_ISSET." );
+    fail_unless(self_pipe_fd_isset(sig, &fds),
+		"Signalled pipe was not FD_ISSET.");
     pthread_join(signal_thread_id, NULL);
 
     self_pipe_destroy(sig);
 }
 END_TEST
 
-
 START_TEST(test_clear_returns_immediately)
 {
     struct self_pipe *sig;
@@ -79,7 +78,6 @@ START_TEST( test_clear_returns_immediately )
 }
 END_TEST
 
-
 START_TEST(test_destroy_closes_read_pipe)
 {
     struct self_pipe *sig;
@@ -116,7 +114,6 @@ START_TEST( test_destroy_closes_read_pipe )
 }
 END_TEST
 
-
 START_TEST(test_destroy_closes_write_pipe)
 {
     struct self_pipe *sig;
@@ -127,7 +124,8 @@ START_TEST( test_destroy_closes_write_pipe )
     orig_write_fd = sig->write_fd;
     self_pipe_destroy(sig);
 
-	while ( ( write_len = write( orig_write_fd, "", 0 ) ) == -1 && errno == EINTR );
+    while ((write_len = write(orig_write_fd, "", 0)) == -1
+	   && errno == EINTR);
 
     switch (write_len) {
     case 0:
@@ -158,8 +156,6 @@ START_TEST( test_destroy_closes_write_pipe )
 }
 END_TEST
 
-
-
 Suite * self_pipe_suite(void)
 {
     Suite *s = suite_create("self_pipe");
@@ -195,4 +191,3 @@ int main(void)
     srunner_free(sr);
     return (number_failed == 0) ? 0 : 1;
 }
-

tests/unit/check_serve.c

@@ -54,7 +54,9 @@ void setup( void )
 
 void teardown(void)
 {
-	if( dummy_file ){ unlink( dummy_file ); }
+    if (dummy_file) {
+	unlink(dummy_file);
+    }
     free(dummy_file);
     dummy_file = NULL;
 }
@@ -64,7 +66,9 @@ START_TEST( test_replaces_acl )
 {
     struct flexnbd flexnbd;
     flexnbd.signal_fd = -1;
-	struct server * s = server_create( &flexnbd, "127.0.0.1", "0", dummy_file, 0, 0, NULL, 1, 0, 1 );
+    struct server *s =
+	server_create(&flexnbd, "127.0.0.1", "0", dummy_file, 0, 0, NULL,
+		      1, 0, 1);
     struct acl *new_acl = acl_create(0, NULL, 0);
 
     server_replace_acl(s, new_acl);
@@ -74,12 +78,13 @@ START_TEST( test_replaces_acl )
 }
 END_TEST
 
-
 START_TEST(test_signals_acl_updated)
 {
     struct flexnbd flexnbd;
     flexnbd.signal_fd = -1;
-	struct server * s = server_create( &flexnbd, "127.0.0.1", "0", dummy_file, 0, 0, NULL, 1, 0, 1 );
+    struct server *s =
+	server_create(&flexnbd, "127.0.0.1", "0", dummy_file, 0, 0, NULL,
+		      1, 0, 1);
     struct acl *new_acl = acl_create(0, NULL, 0);
 
     server_replace_acl(s, new_acl);
@@ -90,7 +95,6 @@ START_TEST( test_signals_acl_updated )
 }
 END_TEST
 
-
 int connect_client(char *addr, int actual_port, char *source_addr)
 {
     int client_fd = -1;
@@ -104,12 +108,15 @@ int connect_client( char *addr, int actual_port, char *source_addr )
     memset(&hint, '\0', sizeof(struct addrinfo));
     hint.ai_socktype = SOCK_STREAM;
 
-	myfail_if( getaddrinfo( addr, NULL, &hint, &ailist ) != 0, "getaddrinfo failed." );
+    myfail_if(getaddrinfo(addr, NULL, &hint, &ailist) != 0,
+	      "getaddrinfo failed.");
 
     int connected = 0;
     for (aip = ailist; aip; aip = aip->ai_next) {
-		((struct sockaddr_in *)aip->ai_addr)->sin_port = htons( actual_port );
+	((struct sockaddr_in *) aip->ai_addr)->sin_port =
-		client_fd = socket( aip->ai_family, aip->ai_socktype, aip->ai_protocol );
+	    htons(actual_port);
+	client_fd =
+	    socket(aip->ai_family, aip->ai_socktype, aip->ai_protocol);
 
 	if (source_addr) {
 	    struct sockaddr src;
@@ -120,7 +127,9 @@ int connect_client( char *addr, int actual_port, char *source_addr )
 	    bind(client_fd, &src, sizeof(struct sockaddr_in6));
 	}
 
-		if( client_fd == -1) { continue; }
+	if (client_fd == -1) {
+	    continue;
+	}
 	if (connect(client_fd, aip->ai_addr, aip->ai_addrlen) == 0) {
 	    connected = 1;
 	    break;
@@ -148,7 +157,9 @@ START_TEST( test_acl_update_closes_bad_client )
      */
     struct flexnbd flexnbd;
     flexnbd.signal_fd = -1;
-	struct server * s = server_create( &flexnbd, "127.0.0.7", "0", dummy_file, 0, 0, NULL, 1, 0, 1 );
+    struct server *s =
+	server_create(&flexnbd, "127.0.0.7", "0", dummy_file, 0, 0, NULL,
+		      1, 0, 1);
     struct acl *new_acl = acl_create(0, NULL, 1);
     struct client *c;
     struct client_tbl_entry *entry;
@@ -187,13 +198,14 @@ START_TEST( test_acl_update_closes_bad_client )
 }
 END_TEST
 
-
 START_TEST(test_acl_update_leaves_good_client)
 {
     struct flexnbd flexnbd;
     flexnbd.signal_fd = -1;
 
-	struct server * s = server_create( &flexnbd, "127.0.0.7", "0", dummy_file, 0, 0, NULL, 1, 0, 1 );
+    struct server *s =
+	server_create(&flexnbd, "127.0.0.7", "0", dummy_file, 0, 0, NULL,
+		      1, 0, 1);
 
     char *lines[] = { "127.0.0.1" };
     struct acl *new_acl = acl_create(1, lines, 1);
@@ -230,7 +242,6 @@ START_TEST( test_acl_update_leaves_good_client )
 }
 END_TEST
 
-
 Suite * serve_suite(void)
 {
     Suite *s = suite_create("serve");
@@ -241,8 +252,10 @@ Suite* serve_suite(void)
     tcase_add_test(tc_acl_update, test_replaces_acl);
     tcase_add_test(tc_acl_update, test_signals_acl_updated);
 
-	tcase_add_exit_test(tc_acl_update, test_acl_update_closes_bad_client, 0);
+    tcase_add_exit_test(tc_acl_update, test_acl_update_closes_bad_client,
-	tcase_add_exit_test(tc_acl_update, test_acl_update_leaves_good_client, 0);
+			0);
+    tcase_add_exit_test(tc_acl_update, test_acl_update_leaves_good_client,
+			0);
 
     suite_add_tcase(s, tc_acl_update);
 
@@ -261,4 +274,3 @@ int main(void)
     srunner_free(sr);
     return (number_failed == 0) ? 0 : 1;
 }
-