Added mirror write barrier / final pass stuff & clean exit afterwards.

Plenty of code documentation.

control.c

@@ -1,3 +1,30 @@
+/*  FlexNBD server (C) Bytemark Hosting 2012
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+/** The control server responds on a UNIX socket and services our "remote"
+ *  commands which are used for changing the access control list, initiating
+ *  a mirror process, or asking for status.  The protocol is pretty simple -
+ *  after connecting the client sends a series of LF-terminated lines, followed
+ *  by a blank line (i.e. double LF).  The first line is taken to be the command
+ *  name to invoke, and the lines before the double LF are its arguments.
+ *
+ *  These commands can be invoked remotely from the command line, with the 
+ *  client code to be found in remote.c
+ */
+
 #include "params.h"
 #include "util.h"
 #include "ioutil.h"
@@ -10,33 +37,74 @@
 #include <sys/un.h>
 #include <unistd.h>
 
-static const int longest_run = 8<<20;
+/** The mirror code will split NBD writes, making them this long as a maximum */
+static const int mirror_longest_write = 8<<20;
 
+/** If, during a mirror pass, we have sent this number of bytes or fewer, we
+ *  go to freeze the I/O and finish it off.  This is just a guess.
+ */
+static const int mirror_last_pass_after_bytes_written = 100<<20;
+
+/** The largest number of full passes we'll do - the last one will always 
+ *  cause the I/O to freeze, however many bytes are left to copy.
+ */
+static const int mirror_maximum_passes = 7;
+
+/** Thread launched to drive mirror process */
 void* mirror_runner(void* serve_params_uncast)
 {
-	struct mode_serve_params *serve = (struct mode_serve_params*) serve_params_uncast;
-	
-	const int max_passes = 7; /* biblical */
+	const int last_pass = mirror_maximum_passes-1;
 	int pass;
+	struct mode_serve_params *serve = (struct mode_serve_params*) serve_params_uncast;
 	struct bitset_mapping *map = serve->mirror->dirty_map;
 	
-	for (pass=0; pass < max_passes; pass++) {
+	for (pass=0; pass < mirror_maximum_passes; pass++) {
 		uint64_t current = 0;
 		uint64_t written = 0;
 		
 		debug("mirror start pass=%d", pass);
 		
+		if (pass == last_pass) {
+			/* last pass, stop everything else */
+			SERVER_ERROR_ON_FAILURE(
+				pthread_mutex_lock(&serve->l_accept),
+				"Problem with accept lock"
+			);
+			SERVER_ERROR_ON_FAILURE(
+				pthread_mutex_lock(&serve->l_io),
+				"Problem with I/O lock"
+			);
+		}
+		
 		while (current < serve->size) {
 			int run;
 		
-			run = bitset_run_count(map, current, longest_run);
+			run = bitset_run_count(map, current, mirror_longest_write);
 			
 			debug("mirror current=%ld, run=%d", current, run);
 			
+			/* FIXME: we could avoid sending sparse areas of the 
+			 * disc here, and probably save a lot of bandwidth and
+			 * time (if we know the destination starts off zeroed).
+			 */ 
 			if (bitset_is_set_at(map, current)) {
+				/* We've found a dirty area, send it */
 				debug("^^^ writing");
 				
-				/* dirty area */
+				/* We need to stop the main thread from working 
+				 * because it might corrupt the dirty map.  This
+				 * is likely to slow things down but will be
+				 * safe.
+				 */
+				if (pass < last_pass)
+					SERVER_ERROR_ON_FAILURE(
+						pthread_mutex_lock(&serve->l_io),
+						"Problem with I/O lock"
+					);
+				
+				/** FIXME: do something useful with bytes/second */
+				
+				/** FIXME: error handling code here won't unlock */
 				socket_nbd_write(
 					serve->mirror->client, 
 					current,
@@ -45,21 +113,60 @@ void* mirror_runner(void* serve_params_uncast)
 					serve->mirror->mapped + current
 				);
 				
+				/* now mark it clean */
 				bitset_clear_range(map, current, run);
+				
+				if (pass < last_pass)
+					SERVER_ERROR_ON_FAILURE(
+						pthread_mutex_unlock(&serve->l_io),
+						"Problem with I/O unlock"
+					);
+				
 				written += run;
 			}
 			current += run;
 		}
 		
-		if (written == 0)
-			pass = max_passes-1;
+		/* if we've not written anything */
+		if (written < mirror_last_pass_after_bytes_written)
+			pass = last_pass;
 	}
 	
+	switch (serve->mirror->action_at_finish)
+	{
+	case ACTION_PROXY:
+		debug("proxy!");
+		serve->proxy_fd = serve->mirror->client;
+		/* don't close our file descriptor, we still need it! */
+		break;
+	case ACTION_EXIT:
+		debug("exit!");
+		write(serve->close_signal[1], serve, 1); /* any byte will do */
+		/* fall through */
+	case ACTION_NOTHING:
+		debug("nothing!");
+		close(serve->mirror->client);
+	}
+	
+	free(serve->mirror->dirty_map);
+	free(serve->mirror);
+	serve->mirror = NULL; /* and we're gone */
+
+	SERVER_ERROR_ON_FAILURE(
+		pthread_mutex_unlock(&serve->l_accept),
+		"Problem with accept unlock"
+	);
+	SERVER_ERROR_ON_FAILURE(
+		pthread_mutex_unlock(&serve->l_io),
+		"Problem with I/O unlock"
+	);
+	
 	return NULL;
 }
 
 #define write_socket(msg) write(client->socket, (msg "\n"), strlen((msg))+1)
 
+/** Command parser to start mirror process from socket input */
 int control_mirror(struct control_params* client, int linesc, char** lines)
 {
 	off64_t size, remote_size;
@@ -146,9 +253,12 @@ int control_mirror(struct control_params* client, int linesc, char** lines)
 		"Failed to create mirror thread"
 	);
 	
+	write_socket("0: mirror started");
+	
 	return 0;
 }
 
+/** Command parser to alter access control list from socket input */
 int control_acl(struct control_params* client, int linesc, char** lines)
 {
 	int acl_entries = 0, parsed;
@@ -173,11 +283,13 @@ int control_acl(struct control_params* client, int linesc, char** lines)
 	return 0;
 }
 
+/** FIXME: add some useful statistics */
 int control_status(struct control_params* client, int linesc, char** lines)
 {
 	return 0;
 }
 
+/** Master command parser for control socket connections, delegates quickly */
 void* control_serve(void* client_uncast)
 {
 	struct control_params* client = (struct control_params*) client_uncast;