Changed msync to actually sync

This copies xnbd's style

.gitlab-ci.yml

@@ -0,0 +1,27 @@
+stages:
+  - package
+  - publish
+ 
+package:jessie: &package
+  stage: package
+  image: $CI_REGISTRY/docker-images/layers:$DISTRO-deb
+  variables:
+    DISTRO: jessie
+  script:
+    - package
+  artifacts:
+    paths:
+      - pkg/
+
+package:stretch:
+  <<: *package
+  variables:
+    DISTRO: stretch
+
+publish:
+  stage: publish
+  tags: 
+    - shell
+  script:
+    - publish
+

Makefile

@@ -11,28 +11,14 @@ ifdef DEBUG
 else
 	CFLAGS_EXTRA=-O2
 endif
+
+NO_MSYNC=1
+ifdef NO_MSYNC
+	CFLAGS_EXTRA += -DNO_MSYNC
+endif
+
 CFLAGS_EXTRA  += -fPIC --std=gnu99
-LDFLAGS_EXTRA += -Wl,--relax,--gc-sections
+LDFLAGS_EXTRA += -Wl,--relax,--gc-sections -L$(LIB) -Wl,-rpath-link,$(LIB)
-
-TOOLCHAIN		:= $(shell $(CC) --version|awk '/Debian/ {print "debian";exit;}')
-#
-# This bit adds extra flags depending of the distro, and the
-# architecture. To make sure debian packages have the right
-# set of 'native' flags on them
-#
-ifeq ($(TOOLCHAIN),debian)
-DEBARCH			:= $(shell dpkg-architecture -qDEB_BUILD_ARCH)
-ifeq ($(DEBARCH),$(filter $(DEBARCH),amd64 i386))
-CFLAGS_EXTRA	+= -march=native
-endif
-ifeq ($(DEBARCH),armhf)
-CFLAGS_EXTRA	+=  -march=armv7-a -mtune=cortex-a8 -mfpu=neon
-endif
-LDFLAGS_EXTRA	+= -L$(LIB) -Wl,-rpath,${shell readlink -f ${LIB}}
-else
-LDFLAGS_EXTRA	+= -L$(LIB) -Wl,-rpath-link,$(LIB)
-endif
-
 
 # The -Wunreachable-code warning is only implemented in clang, but it
 # doesn't break anything for gcc to see it.
@@ -42,23 +28,19 @@ WARNINGS=-Wall \
 				 -Wstrict-prototypes \
 				 -Wno-missing-field-initializers \
 				 -Wunreachable-code
+
 CCFLAGS=-D_GNU_SOURCE=1 $(WARNINGS) $(CFLAGS_EXTRA) $(CFLAGS)
 LLDFLAGS=-lm -lrt -lev $(LDFLAGS_EXTRA) $(LDFLAGS)
 
-
 CC?=gcc
 
 LIBS=-lpthread
 INC=-I/usr/include/libev -Isrc/common -Isrc/server -Isrc/proxy
-COMPILE=$(CC) $(INC) -c $(CCFLAGS)
+COMPILE=$(CC) -MMD $(INC) -c $(CCFLAGS)
-SAVEDEP=$(CC) $(INC) -MM $(CCFLAGS)
 LINK=$(CC) $(LLDFLAGS) -Isrc $(LIBS)
 
 LIB=build/
 
-EXISTING_OBJS := $(wildcard build/*.o)
--include $(EXISTING_OBJS:.o=.d)
-
 COMMON_SRC  := $(wildcard src/common/*.c)
 SERVER_SRC := $(wildcard src/server/*.c)
 PROXY_SRC   := $(wildcard src/proxy/*.c)
@@ -71,12 +53,13 @@ SRCS := $(COMMON_SRC) $(SERVER_SRC) $(PROXY_SRC)
 OBJS := $(COMMON_OBJ) $(SERVER_OBJ) $(PROXY_OBJ)
 
 
-all: build/flexnbd build/flexnbd-proxy doc
+all: build doc
+
+build: server proxy
 
 build/%.o: %.c
 	mkdir -p $(dir $@)
 	$(COMPILE) $< -o $@
-	$(SAVEDEP) $< > build/$*.d
 
 objs: $(OBJS)
 
@@ -87,41 +70,42 @@ build/flexnbd-proxy: $(COMMON_OBJ) $(PROXY_OBJ) build/proxy-main.o
 	$(LINK) $^ -o $@
 
 server: build/flexnbd
+
 proxy: build/flexnbd-proxy
 
-
 CHECK_SRC := $(wildcard tests/unit/*.c)
-CHECK_OBJ := $(CHECK_SRC:tests/unit/%.c=build/tests/%.o)
+CHECK_OBJ := $(CHECK_SRC:tests/unit/%.c=build/%.o)
 # Why can't we reuse the build/%.o rule above? Not sure.
-build/tests/%.o: tests/unit/%.c
-	mkdir -p $(dir $@)
-	$(COMPILE) $< -o $@
-	$(SAVEDEP) $< > build/tests/$*.d
 
-CHECK_BINS := $(CHECK_OBJ:build/tests/%.o=build/tests/%)
+CHECK_BINS := $(CHECK_SRC:tests/unit/%.c=build/%)
-build/tests/%: build/tests/%.o $(OBJS)
+
-	$(LINK) $^ -o $@ -lcheck
+build/check_%: build/check_%.o
+	$(LINK) $^ -o $@ $(COMMON_OBJ) $(SERVER_OBJ) -lcheck -lsubunit
 
 check_objs: $(CHECK_OBJ)
 
 check_bins: $(CHECK_BINS)
-check: $(CHECK_BINS)
+
-	for bin in $^; do $$bin; done
+check: $(OBJS) $(CHECK_BINS)
+	r=true ; for bin in $(CHECK_BINS); do $$bin || r=false; done ; $$r
+
+acceptance: build
+	cd tests/acceptance && RUBYOPT='-I.' ruby nbd_scenarios -v
+
+test: check acceptance
 
 build/flexnbd.1: README.txt
-	a2x --destination-dir build --format manpage $<
+	txt2man -t flexnbd -s 1 $< > $@
+
 build/flexnbd-proxy.1: README.proxy.txt
-	a2x --destination-dir build --format manpage $<
+	txt2man -t flexnbd-proxy -s 1 $< > $@
+
 # If we don't pipe to file, gzip clobbers the original, causing make
 # to rebuild each time
 %.1.gz: %.1
 	gzip -c -f $< > $@
 
-
+doc: build/flexnbd.1.gz build/flexnbd-proxy.1.gz
-server-man: build/flexnbd.1.gz
-proxy-man: build/flexnbd-proxy.1.gz
-
-doc: server-man proxy-man
 
 install:
 	mkdir -p $(INSTALLDIR)
@@ -131,4 +115,7 @@ clean:
 	rm -rf build/*
 
 
-.PHONY: clean objs check_objs all server proxy check_bins check server-man proxy-man doc
+.PHONY: clean objs check_objs all server proxy check_bins check doc build test acceptance
+
+# Include extra dependencies at the end, NOT before 'all'
+-include $(wildcard build/*.d)

README.proxy.txt

@@ -1,19 +1,14 @@
-FLEXNBD-PROXY(1)
-================
-:doctype: manpage
-
 NAME
-----
 
-flexnbd-proxy - A simple NBD proxy
+ flexnbd-proxy - A simple NBD proxy
 
 SYNOPSIS
---------
 
-*flexnbd-proxy* ['OPTIONS']
+  flexnbd-proxy --addr ADDR [--port PORT] --conn-addr ADDR
+    --conn-port PORT [--bind ADDR] [--cache[=CACHE_BYTES]] 
+    [--help] [--verbose] [--quiet]
 
 DESCRIPTION
------------
 
 flexnbd-proxy is a simple NBD proxy server that implements resilient
 connection logic for the client. It connects to an upstream NBD server
@@ -25,11 +20,6 @@ of view of the client) reconnects and retransmits the request, before
 returning the response to the client.
 
 USAGE
------
-
-  $ flexnbd-proxy --addr <ADDR> [ --port <PORT> ]
-    --conn-addr <ADDR> --conn-port <PORT> 
-    [--bind <ADDR>] [--cache[=<CACHE_BYTES>]] [option]*
 
 Proxy requests from an NBD client to an NBD server, resiliently. Only one
 client can be connected at a time, and ACLs cannot be applied to the client, as they 
@@ -58,40 +48,39 @@ Only one request may be in-flight at a time under the current architecture; that
 doesn't seem to slow things down much relative to alternative options, but may
 be changed in the future if it becomes an issue.
 
-Options
+OPTIONS
-~~~~~~~
 
-*--addr, -l ADDR*:
+  --addr, -l ADDR  
     The address to listen on. If this begins with a '/', it is assumed to be
     a UNIX domain socket to create. Otherwise, it should be an IPv4 or IPv6
     address.
-*--port, -p PORT*:
+
+  --port, -p PORT  
     The port to listen on, if --addr is not a UNIX socket. 
 
-*--conn-addr, -C ADDR*:
+  --conn-addr, -C ADDR  
     The address of the NBD server to connect to. Required.
 
-*--conn-port, -P PORT*:
+  --conn-port, -P PORT  
     The port of the NBD server to connect to. Required.
 
-*--cache, -c=CACHE_BYTES*:
+  --cache, -c=CACHE_BYTES  
     If given, the size in bytes of read cache to use. CACHE_BYTES
     defaults to 4096.
 
-*--help, -h* :
+  --help, -h  
     Show command or global help.
 
-*--verbose, -v* :
+  --verbose, -v  
     Output all available log information to STDERR.
 
-*--quiet, -q* :
+  --quiet, -q  
     Output as little log information as possible to STDERR.
 
-
 LOGGING
--------
+
-Log output is sent to STDERR.  If --quiet is set, no output will be seen
+Log output is sent to STDERR. If --quiet is set, no output will be
-unless the program termintes abnormally.  If neither --quiet nor
+seen unless the program termintes abnormally. If neither --quiet nor
 --verbose are set, no output will be seen unless something goes wrong
 with a specific request. If --verbose is given, every available log
 message will be seen (which, for a debug build, is many). It is not an
@@ -101,32 +90,31 @@ The log line format is:
 
     <TIMESTAMP>:<LEVEL>:<PID> <THREAD> <SOURCEFILE>:<SOURCELINE>: <MSG>
 
-*TIMESTAMP*:
+<TIMESTAMP>  
   Time the log entry was made. This is expressed in terms of monotonic ms
 
-*LEVEL*:
+<LEVEL>  
   This will be one of 'D', 'I', 'W', 'E', 'F' in increasing order of
-severity.  If flexnbd is started with the --quiet flag, only 'F' will be
+  severity. If flexnbd is started with the --quiet flag, only 'F' will
-seen.  If it is started with the --verbose flag, any from 'I' upwards
+  be seen. If it is started with the --verbose flag, any from 'I'
-will be seen.  Only if you have a debug build and start it with
+  upwards will be seen. Only if you have a debug build and start it
---verbose will you see 'D' entries.
+  with --verbose will you see 'D' entries.
 
-*PID*:
+<PID>  
   This is the process ID.
 
-*THREAD*:
+<THREAD>  
-  flexnbd-proxy is currently single-threaded, so this should be the same
+  flexnbd-proxy is currently single-threaded, so this should be the
-for all lines. That may not be the case in the future.
+  same for all lines. That may not be the case in the future.
 
-*SOURCEFILE:SOURCELINE*:
+<SOURCEFILE:SOURCELINE>  
   Identifies where in the source code this log line can be found.
 
-*MSG*:
+<MSG>  
   A short message describing what's happening, how it's being done, or
-if you're very lucky *why* it's going on.
+  if you're very lucky why it's going on.
 
-Proxying
+EXAMPLES
-~~~~~~~~
 
 The main point of the proxy mode is to allow clients that would otherwise break
 when the NBD server goes away (during a migration, for instance) to see a
@@ -160,7 +148,6 @@ The data in myfile has been moved between physical servers without the nbd
 client process having to be disturbed at all.
 
 READ CACHE
-----------
 
 If the --cache option is given at the command line, either without an
 argument or with an argument greater than 0, flexnbd-proxy will use a
@@ -173,7 +160,7 @@ cache without making a request to the server.
 This pattern is designed to match sequential reads, such as those
 performed by a booting virtual machine.
 
-Note: If specifying a cache size, you *must* use this form:
+Note: If specifying a cache size, you must use this form:
 
   nbd-client$ flexnbd-proxy --cache=XXXX
 
@@ -183,30 +170,37 @@ If no cache size is given, a size of 4096 bytes is assumed. Caching can
 be explicitly disabled by setting a size of 0.
 
 BUGS
-----
 
-Should be reported to nick@bytemark.co.uk.
+Should be reported via GitHub.
+
+* https://github.com/BytemarkHosting/flexnbd-c/issues
 
 Current issues include:
 
-* Only old-style NBD negotiation is supported
+* only old-style NBD negotiation is supported;
-* Only one request may be in-flight at a time
+* only one request may be in-flight at a time;
-* All I/O is blocking, and signals terminate the process immediately
+* all I/O is blocking, and signals terminate the process immediately;
-* UNIX socket support is limited to the listen address
+* UNIX socket support is limited to the listen address;
-* FLUSH and TRIM commands, and the FUA flag, are not supported
+* FLUSH and TRIM commands, and the FUA flag, are not supported;
-* DISCONNECT requests do not get passed through to the NBD server
+* DISCONNECT requests do not get passed through to the NBD server;
-* No active timeout-retry of requests - we trust the kernel's idea of failure
+* no active timeout-retry of requests - we trust the kernel's idea of
+  failure.
 
 AUTHOR
-------
+
-Written by Alex Young <alex@bytemark.co.uk>.
+Originally written by Alex Young <alex@blackkettle.org>.
 Original concept and core code by Matthew Bloch <matthew@bytemark.co.uk>.
-Proxy mode written by Nick Thomas <nick@bytemark.co.uk>
+Proxy mode written by Nick Thomas <me@ur.gs>.
 
-COPYING
+The full commit history is available on GitHub.
--------
 
-Copyright (c) 2012 Bytemark Hosting Ltd. Free use of this software is
+SEE ALSO
-granted under the terms of the GNU General Public License version 3 or
+
-later.
+flexnbd(1), nbd-client(8), xnbd-server(8), xnbd-client(8)
+
+COPYRIGHT
+
+Copyright (c) 2012-2016 Bytemark Hosting Ltd. Free use of this
+software is granted under the terms of the GNU General Public License
+version 3 or later.
 

README.txt

@@ -1,17 +1,36 @@
-FLEXNBD(1)
-==========
-:doctype: manpage
-
 NAME
-----
+
 flexnbd - A fast NBD server
 
 SYNOPSIS
---------
+
-*flexnbd* 'COMMAND' ['OPTIONS']
+  flexnbd MODE [ ARGS ]
+
+  flexnbd serve --addr ADDR --port PORT --file FILE [--sock SOCK]
+    [--default-deny] [--killswitch] [global_option]* [acl_entry]*
+
+  flexnbd listen --addr ADDR --port PORT --file FILE [--sock SOCK]
+    [--default-deny] [global_option]* [acl_entry]*
+
+  flexnbd mirror --addr ADDR --port PORT --sock SOCK [--unlink]
+    [--bind BIND_ADDR] [global_option]*
+
+  flexnbd acl --sock SOCK [acl_entry]+ [global_option]*
+
+  flexnbd break --sock SOCK [global_option]*
+
+  flexnbd status --sock SOCK [global_option]*
+
+  flexnbd read --addr ADDR --port PORT --from OFFSET --size SIZE
+    [--bind BIND_ADDR] [global_option]*
+
+  flexnbd write --addr ADDR --port PORT --from OFFSET --size SIZE
+    [--bind BIND_ADDR] [global_option]*
+
+  flexnbd help [mode] [global_option]*
 
 DESCRIPTION
------------
+
 Flexnbd is a fast NBD server which supports live migration. Live
 migration is performed by writing the data to a new server. A failed
 migration will be invisible to any connected clients.
@@ -19,57 +38,56 @@ migration will be invisible to any connected clients.
 Flexnbd tries quite hard to preserve sparsity of files it is serving,
 even across migrations.
 
-COMMANDS
+SERVE MODE
---------
+
+Serve a file.
 
-serve
-~~~~~
   $ flexnbd serve --addr <ADDR> --port <PORT> --file <FILE>
-    [--sock <SOCK>] [--default-deny] [-k] [global option]* [acl entry]*
+      [--sock <SOCK>] [--default-deny] [-k] [global_option]*
+      [acl_entry]*
 
-Serve a file. If any ACL entries are given (which should be IP
+If any ACL entries are given (which should be IP
 addresses), only those clients listed will be permitted to connect.
 
 flexnbd will continue to serve until a SIGINT, SIGQUIT, or a successful
 migration.
 
-Options
+  OPTIONS
-^^^^^^^
 
-*--addr, -l ADDR*:
+  --addr, -l ADDR  
     The address to listen on. Required.
 
-*--port, -p PORT*:
+  --port, -p PORT  
     The port to listen on. Required.
 
-*--file, -f FILE*:
+  --file, -f FILE  
     The file to serve. Must already exist. Required.
 
-*--sock, -s SOCK*:
+  --sock, -s SOCK  
     Path to a control socket to open. You will need this if you want to
     migrate, get the current status, or manipulate the access control
     list.
 
-*--default-deny, -d*:
+  --default-deny, -d  
     How to interpret an empty ACL. If --default-deny is given, an
     empty ACL will let no clients connect. If it is not given, an
     empty ACL will let any client connect.
 
-*--killswitch, -k*:
+  --killswitch, -k  
     If set, we implement a 2-minute timeout on NBD requests and
     responses. If a request takes longer than that to complete,
     the client is disconnected. This is useful to keep broken
     clients from breaking migrations, among other things.
 
-listen
+LISTEN MODE
-~~~~~~
-
-  $ flexnbd listen --addr <ADDR> --port <PORT> --file <FILE>
-    [--sock <SOCK>] [--default-deny] [global option]* [acl entry]*
 
 Listen for an inbound migration, and quit with a status of 0 on
 completion.
 
+  $ flexnbd listen --addr ADDR --port PORT --file FILE
+      [--sock SOCK] [--default-deny] [global_option]* 
+      [acl_entry]*
+
 flexnbd will wait for a successful migration, and then quit. The file
 to write the inbound migration data to must already exist before you
 run 'flexnbd listen'.
@@ -81,25 +99,24 @@ to assume that a partial migration can be resumed because the
 destination has no knowledge of whether a client has made a write to
 the source in the interim.
 
-If the migration fails for a reason which the `flexnbd listen` process
+If the migration fails for a reason which the 'flexnbd listen' process
 can't fix (say, a failed local write), it will exit with an error
 status. In this case, the sender will continually retry the migration
-until it succeeds, and you will need to restart the `flexnbd listen`
+until it succeeds, and you will need to restart the 'flexnbd listen'
 process to allow that to happen.
 
-Options
+  OPTIONS
-^^^^^^^
-As for 'serve'.
 
-mirror
+As for serve.
-~~~~~~
 
-  $ flexnbd mirror --addr <ADDR> --port <PORT> --sock SOCK
+MIRROR MODE
-      [--unlink] [--bind <BIND-ADDR>] [global option]*
 
 Start a migration from the server with control socket SOCK to the server
 listening at ADDR:PORT.
 
+  $ flexnbd mirror --addr ADDR --port PORT --sock SOCK [--unlink]
+      [--bind BIND_ADDR] [global_option]*
+
 Migration can be a slow process. Rather than block the 'flexnbd mirror'
 process until it completes, it will exit with a message of "Migration
 started" once it has confirmation that the local server was able to
@@ -112,7 +129,7 @@ again.  It is not safe to resume the migration from where it left off
 because the source can't see that the backing store behind the
 destination is intact, or even on the same machine.
 
-If the `--unlink` option is given, the local file will be deleted
+If the --unlink option is given, the local file will be deleted
 immediately before the mirror connection is terminated. This allows
 an otherwise-ambiguous situation to be resolved: if you don't unlink
 the file and the flexnbd process at either end is terminated, it's not
@@ -122,161 +139,150 @@ the data, there can be no ambiguity.
 
 Note: files smaller than 4096 bytes cannot be mirrored.
 
-Options
+  OPTIONS
-^^^^^^^
 
-*--addr, -l ADDR*:
+  --addr, -l ADDR  
     The address of the remote server to migrate to. Required.
 
-*--port, -p PORT*:
+  --port, -p PORT  
     The port of the remote server to migrate to. Required.
 
-*--sock, -s SOCK*:
+  --sock, -s SOCK  
     The control socket of the local server to migrate from. Required.
 
-*--unlink, -u*:
+  --unlink, -u  
-  Unlink the served file from the local filesystem after successfully
+    Unlink the served file from the local filesystem after
-  mirroring.
+    successfully mirroring.
 
-*--bind, -b BIND-ADDR*:
+  --bind, -b BIND_ADDR  
-  The local address to bind to. You may need this if the remote server
+    The local address to bind to. You may need this if the remote
-  is using an access control list.
+    server is using an access control list.
 
-break
+BREAK MODE
-~~~~~
-
-  $ flexnbd mirror --sock SOCK [global option]*
 
 Stop a running migration.
 
-Options
+  $ flexnbd break --sock SOCK [global_option]*
-^^^^^^^
 
-*--sock, -s SOCK*:
+  OPTIONS
-  The control socket of the local server whose emigration to stop.
+
+  --sock, -s SOCK  
+    The control socket of the local server whose migration to stop.
     Required.
 
-
+ACL MODE
-acl
-~~~
-
-  $ flexnbd acl --sock <SOCK> [acl entry]+ [global option]*
 
 Set the access control list of the server with the control socket SOCK
 to the given access control list entries.
 
+  $ flexnbd acl --sock SOCK [acl_entry]+ [global_option]*
+
 ACL entries are given as IP addresses.
 
-Options
+  OPTIONS
-^^^^^^^
 
-*--sock, -s SOCK*:
+  --sock, -s SOCK  
-  The control socket of the server whose ACL to replace.
+    The control socket of the server whose ACL to replace. Required
 
-status
+STATUS MODE
-~~~~~~
-
-  $ flexnbd status --sock <SOCK> [global option]*
 
 Get the current status of the server with control socket SOCK.
 
+  $ flexnbd status --sock SOCK [global_option]*
+
 The status will be printed to STDOUT. It is a space-separated list of
 key=value pairs. The space character will never appear in a key or
 value. Currently reported values are:
 
-*pid*:
+pid  
   The process id of the server listening on SOCK.
 
-*is_mirroring*:
+is_mirroring  
   'true' if this server is sending migration data, 'false' otherwise.
 
-*has_control*:
+has_control  
   'false' if this server was started in 'listen' mode. 'true' otherwise.
 
-read
+  OPTIONS
-~~~~
 
-  $ flexnbd read --addr <ADDR> --port <PORT> --from <OFFSET>
+  --sock, -s SOCK  
-    --size <SIZE>  [--bind BIND-ADDR] [global option]*
+    The control socket of the server of interest. Required.
+
+READ MODE
 
 Connect to the server at ADDR:PORT, and read SIZE bytes starting at
-OFFSET in a single NBD query.  The returned data will be echoed to
+OFFSET in a single NBD query.
-STDOUT.  In case of a remote ACL, set the local source address to
-BIND-ADDR.
 
-Options
+  $ flexnbd read --addr ADDR --port PORT --from OFFSET --size SIZE
-^^^^^^^
+      [--bind BIND_ADDR] [global_option]*
 
-*--addr, -l ADDR*:
+The returned data will be echoed to STDOUT. In case of a remote ACL,
+set the local source address to BIND_ADDR.
+
+  OPTIONS
+
+  --addr, -l ADDR  
     The address of the remote server. Required.
 
-*--port, -p PORT*:
+  --port, -p PORT  
     The port of the remote server. Required.
 
-*--from, -F OFFSET*:
+  --from, -F OFFSET  
     The byte offset to start reading from. Required.  Maximum 2^62.
 
-*--size, -S SIZE*:
+  --size, -S SIZE  
     The number of bytes to read. Required. Maximum 2^30.
 
-*--bind, -b BIND-ADDR*:
+  --bind, -b BIND_ADDR  
-  The local address to bind to. You may need this if the remote server
+    The local address to bind to. You may need this if the remote
-  is using an access control list.
+    server is using an access control list.
 
-write
+WRITE MODE
-~~~~~
-
-  $ cat ... | flexnbd write --addr <ADDR> --port <PORT> --from <OFFSET>
-    --size <SIZE>  [--bind BIND-ADDR] [global option]*
 
 Connect to the server at ADDR:PORT, and write SIZE bytes from STDIN
-starting at OFFSET in a single NBD query.  In case of a remote ACL, set
+starting at OFFSET in a single NBD query.
-the local source address to BIND-ADDR.
 
-Options
+  $ cat ... | flexnbd write --addr ADDR --port PORT --from OFFSET
-^^^^^^^
+      --size SIZE  [--bind BIND_ADDR] [global_option]*
 
-*--addr, -l ADDR*:
+In case of a remote ACL, set the local source address to BIND_ADDR.
+
+  OPTIONS
+
+  --addr, -l ADDR  
     The address of the remote server. Required.
 
-*--port, -p PORT*:
+  --port, -p PORT  
     The port of the remote server. Required.
 
-*--from, -F OFFSET*:
+  --from, -F OFFSET  
     The byte offset to start writing from. Required.  Maximum 2^62.
 
-*--size, -S SIZE*:
+  --size, -S SIZE  
     The number of bytes to write. Required. Maximum 2^30.
 
-*--bind, -b BIND-ADDR*:
+  --bind, -b BIND_ADDR  
-  The local address to bind to. You may need this if the remote server
+    The local address to bind to. You may need this if the remote
-  is using an access control list.
+    server is using an access control list.
 
-help
+HELP MODE
-~~~~
 
-  $ flexnbd help [command] [global option]*
+  $ flexnbd help [mode] [global_option]*
 
-Without 'command', show the list of available commands.  With 'command',
+Without mode, show the list of available modes. With mode, show help for that mode.
-show help for that command.
 
 GLOBAL OPTIONS
---------------
 
-*--help, -h* :
+--help, -h  Show mode or global help.
-    Show command or global help.
 
-*--verbose, -v* :
+--verbose, -v  Output all available log information to STDERR.
-    Output all available log information to STDERR.
-
-*--quiet, -q* :
-    Output as little log information as possible to STDERR.
 
+--quiet, -q  Output as little log information as possible to STDERR.
 
 LOGGING
--------
+
-Log output is sent to STDERR.  If --quiet is set, no output will be seen
+Log output is sent to STDERR. If --quiet is set, no output will be
-unless the program termintes abnormally.  If neither --quiet nor
+seen unless the program termintes abnormally. If neither --quiet nor
 --verbose are set, no output will be seen unless something goes wrong
 with a specific request. If --verbose is given, every available log
 message will be seen (which, for a debug build, is many). It is not an
@@ -284,39 +290,38 @@ error to set both --verbose and --quiet.  The last one wins.
 
 The log line format is:
 
-  <TIMESTAMP>:<LEVEL>:<PID> <THREAD> <SOURCEFILE>:<SOURCELINE>: <MSG>
+  <TIMESTAMP>:<LEVEL>:<PID> <THREAD> <SOURCEFILE:SOURCELINE>: <MSG>
 
-*TIMESTAMP*:
+<TIMESTAMP>  
-  Time the log entry was made. This is expressed in terms of monotonic ms.
+  Time the log entry was made. This is expressed in terms of monotonic
+  ms.
 
-*LEVEL*:
+<LEVEL>  
   This will be one of 'D', 'I', 'W', 'E', 'F' in increasing order of
-severity.  If flexnbd is started with the --quiet flag, only 'F' will be
+  severity. If flexnbd is started with the --quiet flag, only 'F'
-seen.  If it is started with the --verbose flag, any from 'I' upwards
+  will be seen. If it is started with the --verbose flag, any from 'I'
-will be seen.  Only if you have a debug build and start it with
+  upwards will be seen. Only if you have a debug build and start it
---verbose will you see 'D' entries.
+  with --verbose will you see 'D' entries.
 
-*PID*:
+<PID>  
   This is the process ID.
 
-*THREAD*:
+<THREAD>  
-  There are several pthreads per flexnbd process: a main thread, a serve
+  There are several pthreads per flexnbd process: a main thread, a
-thread, a thread per client, and possibly a pair of mirror threads and a
+  serve thread, a thread per client, and possibly a pair of mirror
-control thread.  This field identifies which thread was responsible for
+  threads and a control thread. This field identifies which thread was
-the log line.
+  responsible for the log line.
 
-*SOURCEFILE:SOURCELINE*:
+<SOURCEFILE:SOURCELINE>  
   Identifies where in the source code this log line can be found.
 
-*MSG*:
+<MSG>  
   A short message describing what's happening, how it's being done, or
-if you're very lucky *why* it's going on.
+  if you're very lucky why it's going on.
 
 EXAMPLES
---------
 
-Serving a file
+  SERVING A FILE
-~~~~~~~~~~~~~~
 
 The simplest case is serving a file on the default nbd port:
 
@@ -326,8 +331,7 @@ The simplest case is serving a file on the default nbd port:
   root:x:
   $
 
-Reading server status
+  READING SERVER STATUS
-~~~~~~~~~~~~~~~~~~~~~
 
 In order to read a server's status, we need it to open a control socket.
 
@@ -340,8 +344,7 @@ In order to read a server's status, we need it to open a control socket.
 
 Note that the status output is newline-terminated.
 
-Migrating
+  MIGRATING
-~~~~~~~~~
 
 To migrate, we need to provide a destination file of the right size.
 
@@ -367,8 +370,8 @@ With this knowledge in hand, we can start the migration:
   $ flexnbd mirror --addr 127.0.0.1 --port 4779 \
       --sock /tmp/flex-source.sock
   Migration started
-  [1]  + 9648 done       build/flexnbd serve --addr 0.0.0.0 --port 4778
+  [1]  + 9648 done       flexnbd serve --addr 0.0.0.0 --port 4778
-  [2]  + 9651 done       build/flexnbd listen --addr 0.0.0.0 --port 4779
+  [2]  + 9651 done       flexnbd listen --addr 0.0.0.0 --port 4779
   $
 
 Note that because the file is so small in this case, we see the source
@@ -376,21 +379,25 @@ server quit soon after we start the migration, and the destination
 exited at roughly the same time.
 
 BUGS
-----
 
-Should be reported to alex@bytemark.co.uk.
+Should be reported on GitHub at
+
+ * https://github.com/BytemarkHosting/flexnbd-c/issues
 
 AUTHOR
-------
 
-Written by Alex Young <alex@bytemark.co.uk>.
+Originally written by Alex Young <alex@blackkettle.org>.
 Original concept and core code by Matthew Bloch <matthew@bytemark.co.uk>.
-Some additions by Nick Thomas <nick@bytemark.co.uk>
+Proxy mode written by Nick Thomas <me@ur.gs>.
 
-COPYING
+The full commit history is available on GitHub.
--------
 
-Copyright (c) 2012 Bytemark Hosting Ltd. Free use of this software is
+SEE ALSO
-granted under the terms of the GNU General Public License version 3 or
-later.
 
+flexnbd-proxy(1), nbd-client(8), xnbd-server(8), xnbd-client(8)
+
+COPYRIGHT
+
+Copyright (c) 2012-2016 Bytemark Hosting Ltd. Free use of this
+software is granted under the terms of the GNU General Public License
+version 3 or later.

Rakefile

@@ -1,60 +0,0 @@
-# encoding: utf-8
-
-def make(*targets)
-  sh "make #{targets.map{|t| t.to_s}.join(" ")}"
-end
-
-def maketask( opts )
-  case opts
-  when Symbol
-    maketask opts => opts
-  else
-    opts.each do |name, targets|
-      task( name ){make *[*targets]}
-    end
-  end
-end
-
-
-desc "Build the binary and man page"
-maketask :build => [:all, :doc]
-
-desc "Build just the flexnbd binary"
-maketask :flexnbd => [:server]
-file "build/flexnbd" => :flexnbd
-
-desc "Build just the flexnbd-proxy binary"
-maketask :flexnbd_proxy => [:proxy]
-file "build/flexnbd-proxy" => :flexnbd_proxy
-
-desc "Build just the man page"
-maketask :man => :doc
-
-
-namespace "test" do
-  desc "Run all tests"
-  task 'run' => ["unit", "scenarios"]
-
-  desc "Build C tests"
-  maketask :build => :check_bins
-
-  desc "Run C tests"
-  maketask :unit => :check
-
-  desc "Run NBD test scenarios"
-  task 'scenarios' => ["build/flexnbd", "build/flexnbd-proxy"] do
-    sh "cd tests/acceptance && RUBYOPT='-I.' ruby nbd_scenarios -v"
-  end
-end
-
-
-desc "Remove all build targets, binaries and temporary files"
-maketask :clean
-
-file "debian/changelog" do
-  FileUtils.mkdir_p "debian"
-  sh "hg log --style=changelog.template > debian/changelog"
-end
-
-desc "Generate the changelog"
-task :changelog => "debian/changelog"

debian/changelog

@@ -1,3 +1,78 @@
+flexnbd (0.1.8) UNRELEASED; urgency=medium
+
+  * Set TCP keepalive on sockets so broken connections are reaped (#33, !33,
+    !36)
+  * Add a context to logs to make debugging problems easier (#34, !34)
+
+ -- James Carter <james.carter@bytemark.co.uk>  Thu, 11 Jan 2018 10:05:35 +0000
+
+flexnbd (0.1.7) stable; urgency=medium
+
+  * Return bytes_left in migration statistics.
+
+ -- Chris Elsworth <chris.elsworth@bytemark.co.uk>  Fri, 14 Jul 2017 17:00:38 +0100
+
+flexnbd (0.1.6) stable; urgency=medium
+
+  * Remove lots of per-cpu compiler flags, notably march=native.
+
+ -- Patrick J Cherry <patrick@bytemark.co.uk>  Thu, 13 Apr 2017 12:48:23 +0100
+
+flexnbd (0.1.5) stable; urgency=medium
+
+  * Switched to native packaging.
+
+ -- Patrick J Cherry <patrick@bytemark.co.uk>  Mon, 23 Jan 2017 13:51:58 +0000
+
+flexnbd (0.1.5-1) stable; urgency=medium
+
+  * This fixes the compiler warning pointer-to-int-cast in serve.c.
+  * Fix up "wrong" handle type from char* to uint64_t in check_readwrite test.
+  * fix check_bitset test on 32-bit platforms.
+  * Skip large file test on 32-bit platforms.
+
+ -- Patrick J Cherry <patrick@bytemark.co.uk>  Fri, 07 Oct 2016 12:25:36 +0100
+
+flexnbd (0.1.4-1) stable; urgency=medium
+
+  [ Michel Pollet ]
+  * Simplified Makefile
+  * Makefile: don't include *.d's before 'all'
+
+  [ Patrick J Cherry ]
+  * Added net-tools to the build-deps for testing
+  * Fixed "double-definition of constants" warning
+  * Use a BINARY encoded string when doing read/write comparisons.
+  * Moved all tasks from Rake to Make
+  * Added dpkg-dev to build-deps
+  * Updated manpages, replace a2x with txt2man
+  * Fixed up internal test names (copy/pasta?)
+  * Fixed check_readwrite test to pass correct handle to fd_write_reply
+  * Fix up nbdtypes test to correctly use htobe64
+  * Update Makefile to specify dependencies properly for tests
+
+  [ James F. Carter ]
+  * update tests to reflect changes in handle storage
+  * run tests in gitlab-ci
+  * force binary encoding in a ruby2.1-compatible way
+  * add missing deps on server object files when building check binaries
+  * remove unnecessary step in gitlab-ci
+  * failures in make check now result in an error
+
+ -- Patrick J Cherry <patrick@bytemark.co.uk>  Thu, 06 Oct 2016 15:49:33 +0100
+
+flexnbd (0.1.3-1) stable; urgency=medium
+
+  [ Michel Pollet ]
+  * Close socket fix, might relate to migration crashing
+  * Attempt at fixing bind() bug
+  * Simplified NBD handle comparisons
+  
+  [ Patrick J Cherry ]
+  * Non-maintainer upload.
+
+ -- Patrick J Cherry <patrick@bytemark.co.uk>  Tue, 04 Oct 2016 21:20:27 +0100
+
 flexnbd (0.1.2-1) stable; urgency=medium
 
   * Non-maintainer upload.

debian/control

@@ -2,8 +2,7 @@ Source: flexnbd
 Section: web
 Priority: extra
 Maintainer: Patrick J Cherry <patrick@bytemark.co.uk> 
-Build-Depends: debhelper (>= 7.0.50), ruby, rake, gcc, libev-dev, 
+Build-Depends: debhelper (>= 7.0.50), ruby, gcc, libev-dev, txt2man, check, net-tools, libsubunit-dev, ruby-test-unit
-  asciidoc, libxml2-utils, xsltproc, xmlto, check
 Standards-Version: 3.8.1
 Homepage: https://github.com/BytemarkHosting/flexnbd-c
 

debian/flexnbd.install

@@ -1,5 +1,3 @@
 build/flexnbd             usr/bin
 build/flexnbd-proxy       usr/bin
-build/flexnbd.1.gz        usr/share/man/man1
-build/flexnbd-proxy.1.gz  usr/share/man/man1
 

debian/flexnbd.manpages

@@ -0,0 +1,2 @@
+build/flexnbd.1.gz
+build/flexnbd-proxy.1.gz

debian/source/format

@@ -1 +1 @@
-3.0 (quilt)
+3.0 (native)

src/common/ioutil.c

@@ -73,7 +73,6 @@ int build_allocation_map(struct bitset * allocation_map, int fd)
 	return 1;
 }
 
-
 int open_and_mmap(const char* filename, int* out_fd, uint64_t *out_size, void **out_map)
 {
 	/* 
@@ -101,7 +100,7 @@ int open_and_mmap(const char* filename, int* out_fd, uint64_t *out_size, void **
 		*out_size = size;
 	}
 
-	if (out_map) {
+	if (0) {
 		*out_map = mmap64(NULL, size, PROT_READ|PROT_WRITE, MAP_SHARED,
 		  *out_fd, 0);
 		if (((long) *out_map) == -1) {
@@ -349,3 +348,36 @@ ssize_t iobuf_write( int fd, struct iobuf *iobuf )
 
 	return count;
 }
+
+struct iommap *iommap_alloc(int fd, off64_t from, uint64_t len) {
+
+	off66_t mmap_from = from & ~((off64_t) getpagesize() - 1);
+        uint64_t mmap_len = len + (from - mmap_from);
+        void *mmap_buf = NULL;
+
+	// TODO: Check the error code from mmap64
+	if(mmap_len)
+		mmap_buf = mmap64(NULL, mmap_len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, mmap_from);
+
+	struct iommap *im = xmalloc(sizeof(struct iommap ));
+
+	im->mmap_buf = mmap_buf;
+	im->mmap_len = mmap_len;
+	im->buf = (char *) mmap_buf + from - mmap_from;
+	debug("mmapped file from %ld:%d", mmap_from, mmap_len);
+	return im;
+}
+
+void iommap_sync(struct iommap *im) {
+        if (im->mmap_len)
+		msync(im->mmap_buf, im->mmap_len, MS_SYNC | MS_INVALIDATE);
+	return;
+}
+
+void iommap_free(struct iommap *im) {
+        if (im->mmap_len)
+		munmap(im->mmap_buf, im->mmap_len);
+
+	free(im);
+}
+

src/common/ioutil.h

@@ -11,7 +11,20 @@ struct iobuf {
 ssize_t iobuf_read( int fd, struct iobuf* iobuf, size_t default_size );
 ssize_t iobuf_write( int fd, struct iobuf* iobuf );
 
+#include <inttypes.h>
+
+struct iommap {
+	char *buf;
+	char *mmap_buf;
+	uint64_t mmap_len;
+};
+
+struct iommap *iommap_alloc(int fd, off64_t from, uint64_t len);
+void   iommap_sync(struct iommap *im);
+void   iommap_free(struct iommap *im);
+
 #include "serve.h"
+
 struct bitset; /* don't need whole of bitset.h here */
 
 /** Scan the file opened in ''fd'', set bits in ''allocation_map''  that

src/common/nbdtypes.c

@@ -27,7 +27,7 @@ void nbd_r2h_request( struct nbd_request_raw *from, struct nbd_request * to )
 {
 	to->magic = htobe32( from->magic );
 	to->type = htobe32( from->type );
-	memcpy( to->handle, from->handle, 8 );
+	to->handle.w = from->handle.w;
 	to->from = htobe64( from->from );
 	to->len = htobe32( from->len );
 }
@@ -36,7 +36,7 @@ void nbd_h2r_request( struct nbd_request * from, struct nbd_request_raw * to )
 {
 	to->magic = be32toh( from->magic );
 	to->type = be32toh( from->type );
-	memcpy( to->handle, from->handle, 8 );
+	to->handle.w = from->handle.w;
 	to->from = be64toh( from->from );
 	to->len = be32toh( from->len );
 }
@@ -46,13 +46,13 @@ void nbd_r2h_reply( struct nbd_reply_raw * from, struct nbd_reply * to )
 {
 	to->magic = htobe32( from->magic );
 	to->error = htobe32( from->error );
-	memcpy( to->handle, from->handle, 8 );
+	to->handle.w = from->handle.w;
 }
 
 void nbd_h2r_reply( struct nbd_reply * from, struct nbd_reply_raw * to )
 {
 	to->magic = be32toh( from->magic );
 	to->error = be32toh( from->error );
-	memcpy( to->handle, from->handle, 8 );
+	to->handle.w = from->handle.w;
 }
 

src/common/nbdtypes.h

@@ -16,7 +16,7 @@
 
 
 /* 1MiB is the de-facto standard for maximum size of header + data */
-#define NBD_MAX_SIZE ( 1024 * 1024 )
+#define NBD_MAX_SIZE ( 32 * 1024 * 1024 )
 
 #define NBD_REQUEST_SIZE ( sizeof( struct nbd_request_raw ) )
 #define NBD_REPLY_SIZE   ( sizeof( struct nbd_reply_raw ) )
@@ -24,6 +24,11 @@
 #include <linux/types.h>
 #include <inttypes.h>
 
+typedef union nbd_handle_t {
+	uint8_t b[8];
+	uint64_t w;
+} nbd_handle_t;
+
 /* The _raw types are the types as they appear on the wire.  Non-_raw
  * types are in host-format.
  * Conversion functions are _r2h_ for converting raw to host, and _h2r_
@@ -39,7 +44,7 @@ struct nbd_init_raw {
 struct nbd_request_raw {
 	__be32 magic;
 	__be32 type;    /* == READ || == WRITE  */
-	char handle[8];
+	nbd_handle_t handle;
 	__be64 from;
 	__be32 len;
 } __attribute__((packed));
@@ -47,7 +52,7 @@ struct nbd_request_raw {
 struct nbd_reply_raw {
 	__be32 magic;
 	__be32 error;           /* 0 = ok, else error   */
-	char handle[8];         /* handle you got from request  */
+	nbd_handle_t handle;         /* handle you got from request  */
 };
 
 
@@ -62,7 +67,7 @@ struct nbd_init {
 struct nbd_request {
 	uint32_t magic;
 	uint32_t type;    /* == READ || == WRITE || == DISCONNECT */
-	char handle[8];
+	nbd_handle_t handle;
 	uint64_t from;
 	uint32_t len;
 } __attribute__((packed));
@@ -70,7 +75,7 @@ struct nbd_request {
 struct nbd_reply {
 	uint32_t magic;
 	uint32_t error;           /* 0 = ok, else error   */
-	char handle[8];         /* handle you got from request  */
+	nbd_handle_t handle;      /* handle you got from request  */
 };
 
 void nbd_r2h_init( struct nbd_init_raw * from, struct nbd_init * to );

src/common/readwrite.c

@@ -105,9 +105,7 @@ void fill_request(struct nbd_request *request, int type, uint64_t from, uint32_t
 {
 	request->magic  = htobe32(REQUEST_MAGIC);
 	request->type   = htobe32(type);
-	uint32_t * randa = (uint32_t*)request->handle;
+	request->handle.w = (((uint64_t)rand()) << 32) | ((uint64_t)rand());
-	randa[0] = rand();
-	randa[1] = rand();
 	request->from   = htobe64(from);
 	request->len    = htobe32(len);
 }
@@ -127,7 +125,7 @@ void read_reply(int fd, struct nbd_request *request, struct nbd_reply *reply)
 	if (reply->error != 0) {
 		error("Server replied with error %d", reply->error);
 	}
-	if (strncmp(request->handle, reply->handle, 8) != 0) {
+	if (request->handle.w != reply->handle.w) {
 		error("Did not reply with correct handle");
 	}
 }

src/common/sockutil.c

@@ -68,6 +68,37 @@ int sock_set_reuseaddr( int fd, int optval )
 	return setsockopt( fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval) );
 }
 
+int sock_set_keepalive_params( int fd, int time, int intvl, int probes)
+{
+	if (sock_set_keepalive(fd, 1) ||
+		sock_set_tcp_keepidle(fd, time) ||
+		sock_set_tcp_keepintvl(fd, intvl) ||
+		sock_set_tcp_keepcnt(fd, probes)) {
+		return -1;
+	}
+	return 0;
+}
+
+int sock_set_keepalive( int fd, int optval )
+{
+	return setsockopt( fd, SOL_SOCKET, SO_KEEPALIVE, &optval, sizeof(optval) );
+}
+
+int sock_set_tcp_keepidle( int fd, int optval )
+{
+	return setsockopt( fd, IPPROTO_TCP, TCP_KEEPIDLE, &optval, sizeof(optval) );
+}
+
+int sock_set_tcp_keepintvl( int fd, int optval )
+{
+	return setsockopt( fd, IPPROTO_TCP, TCP_KEEPINTVL, &optval, sizeof(optval) );
+}
+
+int sock_set_tcp_keepcnt( int fd, int optval )
+{
+	return setsockopt( fd, IPPROTO_TCP, TCP_KEEPCNT, &optval, sizeof(optval) );
+}
+
 /* Set the tcp_nodelay option */
 int sock_set_tcp_nodelay( int fd, int optval )
 {
@@ -100,7 +131,7 @@ int sock_try_bind( int fd, const struct sockaddr* sa )
 {
 	int bind_result;
 	char s_address[256];
-	int retry = 1;
+	int retry = 10;
 
 	sockaddr_address_string( sa, &s_address[0], 256 );
 
@@ -126,8 +157,11 @@ int sock_try_bind( int fd, const struct sockaddr* sa )
 				 * will cope with it.
 				 */
 				case EADDRNOTAVAIL:
+					retry--;
+					if (retry) {
 						debug( "retrying" );
 						sleep( 1 );
+					}
 					continue;
 				case EADDRINUSE:
 					warn( "%s in use, giving up.", s_address );

src/common/sockutil.h

@@ -14,9 +14,24 @@ size_t sockaddr_size(const struct sockaddr* sa);
  */
 const char* sockaddr_address_string(const struct sockaddr* sa, char* dest, size_t len);
 
+/* Configure TCP keepalive on a socket */
+int sock_set_keepalive_params( int fd, int time, int intvl, int probes);
+
+/* Set the SOL_KEEPALIVE otion */
+int sock_set_keepalive(int fd, int optval);
+
 /* Set the SOL_REUSEADDR otion */
 int sock_set_reuseaddr(int fd, int optval);
 
+/* Set the tcp_keepidle option */
+int sock_set_tcp_keepidle(int fd, int optval);
+
+/* Set the tcp_keepintvl option */
+int sock_set_tcp_keepintvl(int fd, int optval);
+
+/* Set the tcp_keepcnt option */
+int sock_set_tcp_keepcnt(int fd, int optval);
+
 /* Set the tcp_nodelay option */
 int sock_set_tcp_nodelay(int fd, int optval);
 

src/common/util.c

@@ -13,6 +13,7 @@
 pthread_key_t cleanup_handler_key;
 
 int log_level = 2;
+char *log_context = "";
 
 void error_init(void)
 {

src/common/util.h

@@ -21,6 +21,9 @@ extern int log_level;
 /* set up the error globals */
 void error_init(void);
 
+/* some context for the overall process that appears on each log line */
+extern char *log_context;
+
 
 void exit_err( const char * );
 
@@ -92,7 +95,7 @@ uint64_t monotonic_time_ms(void);
 
 #define levstr(i) (i==0?'D':(i==1?'I':(i==2?'W':(i==3?'E':'F'))))
 
-#define myloglev(level, msg, ...) mylog( level, "%"PRIu64":%c:%d %p %s:%d: "msg"\n", monotonic_time_ms(), levstr(level), getpid(),pthread_self(), __FILE__, __LINE__, ##__VA_ARGS__ )
+#define myloglev(level, msg, ...) mylog( level, "%"PRIu64":%c:%d %p %s %s:%d: "msg"\n", monotonic_time_ms(), levstr(level), getpid(),pthread_self(), log_context, __FILE__, __LINE__, ##__VA_ARGS__ )
 
 #ifdef DEBUG
 #  define debug(msg, ...) myloglev(0, msg, ##__VA_ARGS__)

src/proxy/proxy.c

@@ -79,6 +79,9 @@ struct proxier* proxy_create(
 	out->req.buf  = xmalloc( NBD_MAX_SIZE );
 	out->rsp.buf  = xmalloc( NBD_MAX_SIZE );
 
+	log_context = xmalloc( strlen(s_upstream_address) + strlen(s_upstream_port) + 2 );
+	sprintf(log_context, "%s:%s", s_upstream_address, s_upstream_port);
+
 	return out;
 }
 
@@ -709,7 +712,7 @@ void proxy_session( struct proxier* proxy )
 			state_started = monotonic_time_ms();
 
 			debug(
-				"State transitition from %s to %s",
+				"State transition from %s to %s",
 				proxy_session_state_names[old_state],
 				proxy_session_state_names[state]
 			);

src/server/client.c

@@ -118,6 +118,7 @@ void write_not_zeroes(struct client* client, uint64_t from, uint64_t len)
 		 */
 
 		uint64_t run = bitset_run_count(map, from, len);
+		struct iommap *iommap = iommap_alloc(client->fileno, from, len); 
 
 		debug("write_not_zeroes: from=%ld, len=%d, run=%d", from, len, run);
 
@@ -155,7 +156,7 @@ void write_not_zeroes(struct client* client, uint64_t from, uint64_t len)
 		if (bitset_is_set_at(map, from)) {
 			debug("writing the lot: from=%ld, run=%d", from, run);
 			/* already allocated, just write it all */
-			DO_READ(client->mapped + from, run);
+			DO_READ(iommap->buf, run);
 			/* We know from our earlier call to  bitset_run_count that the
 			 * bitset is all-1s at this point, but we need to dirty it for the
 			 * sake of the event stream - the actual bytes have changed, and we
@@ -186,7 +187,7 @@ void write_not_zeroes(struct client* client, uint64_t from, uint64_t len)
 					(0 == memcmp( zerobuffer, zerobuffer+1, blockrun-1 ));
 
 				if ( !all_zeros ) {
-					memcpy(client->mapped+from, zerobuffer, blockrun);
+					memcpy(iommap->buf, zerobuffer, blockrun);
 					bitset_set_range(map, from, blockrun);
 					/* at this point we could choose to
 					 * short-cut the rest of the write for
@@ -205,6 +206,8 @@ void write_not_zeroes(struct client* client, uint64_t from, uint64_t len)
 				from += blockrun;
 			}
 		}
+		iommap_sync(iommap);
+		iommap_free(iommap);
 	}
 }
 
@@ -234,6 +237,10 @@ int client_read_request( struct client * client , struct nbd_request *out_reques
 				debug( "Connection reset while"
 						" reading request" );
 				return 0;
+			case ETIMEDOUT:
+				debug( "Connection timed out while"
+						" reading request" );
+				return 0;
 			default:
 				/* FIXME: I've seen this happen, but I
 				 * couldn't reproduce it so I'm leaving
@@ -252,14 +259,14 @@ int client_read_request( struct client * client , struct nbd_request *out_reques
 	return 1;
 }
 
-int fd_write_reply( int fd, char *handle, int error )
+int fd_write_reply( int fd, uint64_t handle, int error )
 {
 	struct nbd_reply     reply;
 	struct nbd_reply_raw reply_raw;
 
 	reply.magic = REPLY_MAGIC;
 	reply.error = error;
-	memcpy( reply.handle, handle, 8 );
+	reply.handle.w = handle;
 
 	nbd_h2r_reply( &reply, &reply_raw );
 	debug( "Replying with handle=0x%08X, error=%"PRIu32, handle, error );
@@ -291,7 +298,7 @@ int fd_write_reply( int fd, char *handle, int error )
  */
 int client_write_reply( struct client * client, struct nbd_request *request, int error )
 {
-	return fd_write_reply( client->socket, request->handle, error);
+	return fd_write_reply( client->socket, request->handle.w, error);
 }
 
 
@@ -445,22 +452,28 @@ void client_reply_to_read( struct client* client, struct nbd_request request )
 void client_reply_to_write( struct client* client, struct nbd_request request )
 {
 	debug("request write from=%"PRIu64", len=%"PRIu32", handle=0x%08X", request.from, request.len, request.handle);
-	if (client->serve->allocation_map_built) {
+	// TODO: Just write directly for now.  Not (yet) convinced my changes later on work.
+	// if (client->serve->allocation_map_built) {
+	if (0) {
 		write_not_zeroes( client, request.from, request.len );
 	}
 	else {
 		debug("No allocation map, writing directly.");
 		/* If we get cut off partway through reading this data:
 		 * */
+		struct iommap *iommap = iommap_alloc(client->fileno, request.from, request.len); 
+
 		ERROR_IF_NEGATIVE(
 			readloop( client->socket,
-				client->mapped + request.from,
+				iommap->buf,
 				request.len),
 			"reading write data failed from=%ld, len=%d",
 			request.from,
 			request.len
 		);
 
+		iommap_sync(iommap);
+		iommap_free(iommap);
 		/* the allocation_map is shared between client threads, and may be
 		 * being built. We need to reflect the write in it, as it may be in
 		 * a position the builder has already gone over.
@@ -468,19 +481,6 @@ void client_reply_to_write( struct client* client, struct nbd_request request )
 		bitset_set_range(client->serve->allocation_map, request.from, request.len);
 	}
 
-	if (1) /* not sure whether this is necessary... */
-	{
-		/* multiple of 4K page size */
-		uint64_t from_rounded = request.from & (!0xfff);
-		uint64_t len_rounded = request.len + (request.from - from_rounded);
-
-		FATAL_IF_NEGATIVE(
-			msync( client->mapped + from_rounded,
-				len_rounded,
-				MS_SYNC | MS_INVALIDATE),
-			"msync failed %ld %ld", request.from, request.len
-		);
-	}
 	client_write_reply( client, &request, 0);
 }
 
@@ -646,9 +646,6 @@ void client_cleanup(struct client* client,
 		debug("Closed client socket fd %d", client->socket);
 		client->socket = -1;
 	}
-	if (client->mapped) {
-		munmap(client->mapped, client->serve->size);
-	}
 	if (client->fileno) {
 		FATAL_IF_NEGATIVE( close(client->fileno),
 			"Error closing file %d",
@@ -664,25 +661,20 @@ void client_cleanup(struct client* client,
 void* client_serve(void* client_uncast)
 {
 	struct client* client = (struct client*) client_uncast;
+	void** a = NULL;
 
 	error_set_handler((cleanup_handler*) client_cleanup, client);
-
 	info("client: mmaping file");
 	FATAL_IF_NEGATIVE(
 		open_and_mmap(
 			client->serve->filename,
 			&client->fileno,
 			NULL,
-			(void**) &client->mapped
+			a
 		),
 		"Couldn't open/mmap file %s: %s", client->serve->filename, strerror( errno )
 	);
 
-	FATAL_IF_NEGATIVE(
-		madvise( client->mapped, client->serve->size, MADV_RANDOM ),
-		SHOW_ERRNO( "Failed to madvise() %s", client->serve->filename )
-	);
-
 	debug( "Opened client file fd %d", client->fileno);
 	debug("client: sending hello");
 	client_send_hello(client);

src/server/client.h

@@ -29,7 +29,6 @@ struct client {
 	int     socket;
 
 	int     fileno;
-	char*   mapped;
 
 	struct self_pipe * stop_signal;
 

src/server/mirror.c

@@ -412,7 +412,7 @@ int mirror_setup_next_xfer( struct mirror_ctrl *ctrl )
 	struct nbd_request req = {
 		.magic = REQUEST_MAGIC,
 		.type = REQUEST_WRITE,
-		.handle = ".MIRROR.",
+		.handle.b = ".MIRROR.",
 		.from = current,
 		.len = run
 	};
@@ -568,7 +568,7 @@ static void mirror_read_cb( struct ev_loop *loop, ev_io *w, int revents )
 		return;
 	}
 
-	if ( memcmp( ".MIRROR.", &rsp.handle[0], 8 ) != 0 ) {
+	if ( memcmp( ".MIRROR.", rsp.handle.b, 8 ) != 0 ) {
 		warn( "Bad handle returned from listener" );
 		ev_break( loop, EVBREAK_ONE );
 		return;
@@ -922,7 +922,7 @@ void* mirror_runner(void* serve_params_uncast)
 	 * for us ). But if we've failed and are going to retry on the next run, we
 	 * must close this socket here to have any chance of it succeeding.
 	 */
-	if ( !mirror->client < 0 ) {
+	if ( !(mirror->client < 0) ) {
 		sock_try_close( mirror->client );
 		mirror->client = -1;
 	}

src/server/serve.c

@@ -78,6 +78,8 @@ struct server * server_create (
 	NULLCHECK( out->close_signal );
 	NULLCHECK( out->acl_updated_signal );
 
+	log_context = s_file;
+
 	return out;
 }
 
@@ -255,7 +257,7 @@ int tryjoin_client_thread( struct client_tbl_entry *entry, int (*joinfunc)(pthre
 			debug("nbd thread %016x exited (%s) with status %ld",
 					entry->thread,
 					s_client_address,
-					(uint64_t)status);
+					(uintptr_t)status);
 			client_destroy( entry->client );
 			entry->client = NULL;
 			entry->thread = 0;
@@ -422,6 +424,9 @@ void accept_nbd_client(
 	int slot;
 	char s_client_address[64] = {0};
 
+	FATAL_IF_NEGATIVE( sock_set_keepalive_params( client_fd, CLIENT_KEEPALIVE_TIME, CLIENT_KEEPALIVE_INTVL, CLIENT_KEEPALIVE_PROBES),
+		"Error setting keepalive parameters on client socket fd %d", client_fd );
+
 
 	if ( !server_should_accept_client( params, client_address, s_client_address, 64 ) ) {
 		FATAL_IF_NEGATIVE( close( client_fd ),

src/server/serve.h

@@ -21,6 +21,9 @@ struct client_tbl_entry {
 
 
 #define MAX_NBD_CLIENTS 16
+#define CLIENT_KEEPALIVE_TIME 30
+#define CLIENT_KEEPALIVE_INTVL 10
+#define CLIENT_KEEPALIVE_PROBES 3
 struct server {
 	/* The flexnbd wrapper this server is attached to */
 	struct flexnbd * flexnbd;

src/server/status.c

@@ -31,6 +31,7 @@ struct status * status_create( struct server * serve )
 		status->migration_speed_limit = serve->mirror->max_bytes_per_second;
 
 		status->migration_seconds_left = server_mirror_eta( serve );
+		status->migration_bytes_left = server_mirror_bytes_remaining( serve );
 	}
 
 	server_unlock_start_mirror( serve );
@@ -60,6 +61,7 @@ int status_write( struct status * status, int fd )
 		PRINT_UINT64( migration_speed );
 		PRINT_UINT64( migration_duration );
 		PRINT_UINT64( migration_seconds_left );
+		PRINT_UINT64( migration_bytes_left );
 		if ( status->migration_speed_limit < UINT64_MAX ) {
 			PRINT_UINT64( migration_speed_limit );
 		};

src/server/status.h

@@ -64,6 +64,8 @@
  *   Our current best estimate of how many seconds are left before the migration
  *   migration is finished.
  *
+ * migration_bytes_left:
+ *   The number of bytes remaining to migrate.
  */
 
 
@@ -84,6 +86,7 @@ struct status {
 	uint64_t migration_speed;
 	uint64_t migration_speed_limit;
 	uint64_t migration_seconds_left;
+	uint64_t migration_bytes_left;
 };
 
 /** Create a status object for the given server. */

tests/acceptance/flexnbd/constants.rb

@@ -32,7 +32,7 @@ module FlexNBD
       txt_lines.each do |line|
         if line =~ /^#\s*define\s+([A-Z0-9_]+)\s+(\d+)\s*$/
           # Bodge until I can figure out what to do with #ifdefs
-          const_set($1, $2.to_i) unless constants.include?( $1 )
+          const_set($1, $2.to_i) unless const_defined?( $1 )
         end
       end
     end

tests/acceptance/proxy_tests.rb

@@ -3,6 +3,10 @@ require 'flexnbd/fake_source'
 require 'flexnbd/fake_dest'
 
 module ProxyTests
+  def b
+    "\xFF".b
+  end
+
   def with_proxied_client( override_size = nil )
     @env.serve1 unless @server_up
     @env.proxy2 unless @proxy_up
@@ -51,7 +55,7 @@ module ProxyTests
     with_proxied_client do |client|
       (0..3).each do |n|
         offset = n * 4096
-        client.write(offset, "\xFF" * 4096)
+        client.write(offset, b * 4096)
         rsp = client.read_response
 
         assert_equal FlexNBD::REPLY_MAGIC, rsp[:magic]
@@ -60,7 +64,7 @@ module ProxyTests
 
         data = @env.file1.read(offset, 4096)
 
-        assert_equal( ( "\xFF" * 4096 ), data, "Data not written correctly (offset is #{n})" )
+        assert_equal( ( b * 4096 ), data, "Data not written correctly (offset is #{n})" )
       end
     end
   end
@@ -107,7 +111,7 @@ module ProxyTests
 
       # The reply should be proxied back to the client.
       sc2.write_reply( req2[:handle] )
-      sc2.write_data( "\xFF" * 4096 )
+      sc2.write_data( b * 4096 )
 
       # Check it to make sure it's correct
       rsp = timeout(15) { client.read_response }
@@ -116,7 +120,7 @@ module ProxyTests
       assert_equal req1[:handle], rsp[:handle]
 
       data = client.read_raw( 4096 )
-      assert_equal( ("\xFF" * 4096), data, "Wrong data returned" )
+      assert_equal( (b * 4096), data, "Wrong data returned" )
 
       sc2.close
       server.close
@@ -131,7 +135,7 @@ module ProxyTests
       server, sc1 = maker.value
 
       # Send the read request to the proxy
-      client.write( 0, ( "\xFF" * 4096 ) )
+      client.write( 0, ( b * 4096 ) )
 
       # ensure we're given the read request
       req1 = sc1.read_request
@@ -140,7 +144,7 @@ module ProxyTests
       assert_equal 0, req1[:from]
       assert_equal 4096, req1[:len]
       data1 = sc1.read_data( 4096 )
-      assert_equal( ( "\xFF" * 4096 ), data1, "Data not proxied successfully" )
+      assert_equal( ( b * 4096 ), data1, "Data not proxied successfully" )
 
       # Kill the server again, now we're sure the read request has been sent once
       sc1.close

tests/acceptance/test_happy_path.rb

@@ -115,6 +115,11 @@ class TestHappyPath < Test::Unit::TestCase
 
 
   def test_write_to_high_block
+		# 
+		# This test does not work on 32 bit platforms.
+		#
+		skip("Not relevant on 32-bit platforms") if ( ["a"].pack("p").size < 8 )
+
     # Create a large file, then try to write to somewhere after the 2G boundary
     @env.truncate1 "4G"
     @env.serve1

tests/acceptance/test_serve_mode.rb

@@ -6,6 +6,7 @@ class TestServeMode < Test::Unit::TestCase
 
   def setup
     super
+    @b = "\xFF".b
     @env = Environment.new
     @env.writefile1( "0" )
     @env.serve1
@@ -53,18 +54,18 @@ class TestServeMode < Test::Unit::TestCase
       assert_equal FlexNBD::REPLY_MAGIC, rsp[:magic]
       assert_equal 0, rsp[:error]
 
-      client.write( 0, "\xFF" )
+      client.write( 0, @b )
       rsp = client.read_response
       assert_equal FlexNBD::REPLY_MAGIC, rsp[:magic]
       assert_equal 0, rsp[:error]
 
-      client.write( 0, "\xFF\xFF" )
+      client.write( 0, @b * 2 )
       rsp = client.read_response
       assert_equal FlexNBD::REPLY_MAGIC, rsp[:magic]
       assert_equal 0, rsp[:error]
     end
 
-    assert_equal "\xFF\xFF", @env.file1.read( 0, 2 )
+    assert_equal @b * 2, @env.file1.read( 0, 2 )
   end
 
 

tests/unit/check_bitset.c

@@ -59,7 +59,7 @@ END_TEST
 START_TEST(test_bit_ranges)
 {
 	bitfield_word_t buffer[BIT_WORDS_FOR_SIZE(4160)];
-	uint64_t  *longs = (unsigned long*) buffer;
+	uint64_t  *longs = (uint64_t *) buffer;
 	uint64_t i;
 
 	memset(buffer, 0, 4160);
@@ -67,9 +67,9 @@ START_TEST(test_bit_ranges)
 	for (i=0; i<64; i++) {
 		bit_set_range(buffer, i*64, i);
 		fail_unless(
-			longs[i] == (1UL<<i)-1,
+			longs[i] == (1ULL<<i)-1,
 			"longs[%ld] = %lx SHOULD BE %lx",
-			i, longs[i], (1L<<i)-1
+			i, longs[i], (1ULL<<i)-1
 		);
 
 		fail_unless(longs[i+1] == 0, "bit_set_range overshot at i=%d", i);

tests/unit/check_mbox.c

@@ -66,9 +66,9 @@ START_TEST( test_receive_blocks_until_post )
 END_TEST
 
 
-Suite* acl_suite(void)
+Suite* mbox_suite(void)
 {
-	Suite *s = suite_create("acl");
+	Suite *s = suite_create("mbox");
 	TCase *tc_create = tcase_create("create");
 	TCase *tc_post = tcase_create("post");
 
@@ -93,7 +93,7 @@ int main(void)
 	log_level = 2;
 #endif
 	int number_failed;
-	Suite *s = acl_suite();
+	Suite *s = mbox_suite();
 	SRunner *sr = srunner_create(s);
 	srunner_run_all(sr, CK_NORMAL);
 	log_level = 0;

tests/unit/check_nbdtypes.c

@@ -88,14 +88,14 @@ START_TEST(test_request_handle)
 	struct nbd_request_raw request_raw;
 	struct nbd_request     request;
 
-	memcpy( request_raw.handle, "MYHANDLE", 8 );
+	memcpy( request_raw.handle.b, "MYHANDLE", 8 );
 	
 	nbd_r2h_request( &request_raw, &request );
-	memset( request_raw.handle, 0, 8 );
+	request_raw.handle.w = 0;
 	nbd_h2r_request( &request, &request_raw );
   
-	fail_unless( memcmp( request.handle, "MYHANDLE", 8 ) == 0, "The handle was not copied." );
+	fail_unless( memcmp( request.handle.b, "MYHANDLE", 8 ) == 0, "The handle was not copied." );
-	fail_unless( memcmp( request_raw.handle, "MYHANDLE", 8 ) == 0, "The handle was not copied back." );
+	fail_unless( memcmp( request_raw.handle.b, "MYHANDLE", 8 ) == 0, "The handle was not copied back." );
 }
 END_TEST
 
@@ -170,14 +170,14 @@ START_TEST(test_reply_handle)
 	struct nbd_reply_raw reply_raw;
 	struct nbd_reply     reply;
 
-	memcpy( reply_raw.handle, "MYHANDLE", 8 );
+	memcpy( reply_raw.handle.b, "MYHANDLE", 8 );
 	
 	nbd_r2h_reply( &reply_raw, &reply );
-	memset( reply_raw.handle, 0, 8 );
+	reply_raw.handle.w = 0;
 	nbd_h2r_reply( &reply, &reply_raw );
   
-	fail_unless( memcmp( reply.handle, "MYHANDLE", 8 ) == 0, "The handle was not copied." );
+	fail_unless( memcmp( reply.handle.b, "MYHANDLE", 8 ) == 0, "The handle was not copied." );
-	fail_unless( memcmp( reply_raw.handle, "MYHANDLE", 8 ) == 0, "The handle was not copied back." );
+	fail_unless( memcmp( reply_raw.handle.b, "MYHANDLE", 8 ) == 0, "The handle was not copied back." );
 }
 END_TEST
 
@@ -188,14 +188,15 @@ START_TEST( test_convert_from )
 	 * nbd_request_raw */
 	struct nbd_request_raw request_raw;
 	struct nbd_request     request;
-	char readbuf[] = {0x80, 0, 0, 0, 0, 0, 0, 0};
 
-	memcpy( &request_raw.from, readbuf, 8 );
+	uint64_t target = 0x8000000000000000;
 
+	/* this is stored big-endian */
+	request_raw.from = htobe64(target);
+
+	/* We expect this to convert big-endian to the host format */
 	nbd_r2h_request( &request_raw, &request );
 
-	uint64_t target = 1;
-	target <<= 63;
 	fail_unless( target == request.from, "from was wrong" );
 }	
 END_TEST

tests/unit/check_readwrite.c

@@ -22,7 +22,7 @@
 
 
 int fd_read_request( int, struct nbd_request_raw *);
-int fd_write_reply( int, char *, int );
+int fd_write_reply( int, uint64_t, int );
 
 int marker;
 
@@ -46,8 +46,7 @@ void * responder( void *respond_uncast )
 	struct respond * resp = (struct respond *) respond_uncast;
 	int sock_fd = resp->sock_fds[1];
 	struct nbd_request_raw request_raw;
-	char wrong_handle[] = "WHOOPSIE";
+	uint64_t wrong_handle = 0x80;
-
 
 	if( fd_read_request( sock_fd, &request_raw ) == -1){
 		fprintf(stderr, "Problem with fd_read_request\n");
@@ -57,7 +56,7 @@ void * responder( void *respond_uncast )
 			fd_write_reply( sock_fd, wrong_handle, 0 );
 		}
 		else {
-			fd_write_reply( sock_fd, resp->received.handle, 0 );
+			fd_write_reply( sock_fd, resp->received.handle.w, 0 );
 		}
 		write( sock_fd, "12345678", 8 );
 	}
@@ -150,7 +149,7 @@ END_TEST
 
 Suite* readwrite_suite(void)
 {
-	Suite *s = suite_create("acl");
+	Suite *s = suite_create("readwrite");
 	TCase *tc_transfer = tcase_create("entrust");
 	TCase *tc_disconnect = tcase_create("disconnect");
 

tests/unit/check_status.c

@@ -308,6 +308,7 @@ START_TEST( test_renders_migration_statistics )
 	status.migration_speed = 40000000;
 	status.migration_speed_limit = 40000001;
 	status.migration_seconds_left = 1;
+	status.migration_bytes_left = 5000;
 
 	status_write( &status, fds[1] );
 	fail_if_rendered( fds[0], "migration_duration" );
@@ -335,6 +336,9 @@ START_TEST( test_renders_migration_statistics )
 	status_write( &status, fds[1] );
 	fail_unless_rendered( fds[0], "migration_seconds_left=1" );
 
+	status_write( &status, fds[1] );
+	fail_unless_rendered( fds[0], "migration_bytes_left=5000" );
+
 	status.migration_speed_limit = UINT64_MAX;
 
 	status_write( &status, fds[1] );

tests/unit/check_util.c

@@ -141,9 +141,9 @@ START_TEST( test_fatal_doesnt_call_handler )
 END_TEST
 
 
-Suite* error_suite(void)
+Suite* util_suite(void)
 {
-	Suite *s = suite_create("error");
+	Suite *s = suite_create("util");
 	TCase *tc_process = tcase_create("process");
 	TCase *tc_handler = tcase_create("handler");
 
@@ -163,7 +163,7 @@ Suite* error_suite(void)
 int main(void)
 {
 	int number_failed;
-	Suite *s = error_suite();
+	Suite *s = util_suite();
 	SRunner *sr = srunner_create(s);
 	srunner_run_all(sr, CK_NORMAL);
 	number_failed = srunner_ntests_failed(sr);