Free all possibly held mutexes in error handlers

Now that we have 3 mutexes lying around, it's important that we check
and free these if necessary if error() is called in any thread that can
hold them.  To do this, we now have flexthread.c, which defines a
flexthread_mutex struct.  This is a wrapper around a pthread_mutex_t and
a pthread_t.  The idea is that in the error handler, the thread can
check whether it holds the mutex and can free it if and only if it does.
This is important because pthread fast mutexes can be freed by *any*
thread, not just the thread which holds them.

Note: it is only ever safe for a thread to check if it holds the mutex
itself.  It is *never* safe to check if another thread holds a mutex
without first locking that mutex, which makes the whole operation rather
pointless.

src/serve.c

@@ -81,8 +81,8 @@ struct server * server_create (
 	strcpy(out->filename_incomplete, s_file);
 	strcpy(out->filename_incomplete + strlen(s_file), ".INCOMPLETE");
 
-	pthread_mutex_init(&out->l_io, NULL);
-	pthread_mutex_init(&out->l_acl, NULL);
+	out->l_io = flexthread_mutex_create();
+	out->l_acl= flexthread_mutex_create();
 
 	out->close_signal = self_pipe_create();
 	out->acl_updated_signal = self_pipe_create();
@@ -100,8 +100,8 @@ void server_destroy( struct server * serve )
 	self_pipe_destroy( serve->close_signal );
 	serve->close_signal = NULL;
 
-	pthread_mutex_destroy( &serve->l_acl );
-	pthread_mutex_destroy( &serve->l_io );
+	flexthread_mutex_destroy( serve->l_acl );
+	flexthread_mutex_destroy( serve->l_io );
 
 	if ( serve->acl ) { 
 		acl_destroy( serve->acl );
@@ -126,23 +126,39 @@ void server_dirty(struct server *serve, off64_t from, int len)
 
 #define SERVER_LOCK( s, f, msg ) \
 	do { NULLCHECK( s ); \
-	 FATAL_IF( 0 != pthread_mutex_lock( &s->f ), msg ); } while (0)
+	 FATAL_IF( 0 != flexthread_mutex_lock( s->f ), msg ); } while (0)
 #define SERVER_UNLOCK( s, f, msg ) \
 	do { NULLCHECK( s ); \
-	 FATAL_IF( 0 != pthread_mutex_unlock( &s->f ), msg ); } while (0)
+	 FATAL_IF( 0 != flexthread_mutex_unlock( s->f ), msg ); } while (0)
 
 void server_lock_io( struct server * serve)
 {
+	debug("IO locking");
+
 	SERVER_LOCK( serve, l_io, "Problem with I/O lock" );
 }
 
 void server_unlock_io( struct server* serve )
 {
+	debug("IO unlocking");
+
 	SERVER_UNLOCK( serve, l_io, "Problem with I/O unlock" );
 }
 
+
+/* This is only to be called from error handlers. */
+int server_io_locked( struct server * serve )
+{
+	NULLCHECK( serve );
+	return flexthread_mutex_held( serve->l_io );
+}
+
+
+
 void server_lock_acl( struct server *serve )
 {
+	debug("ACL locking");
+
 	SERVER_LOCK( serve, l_acl, "Problem with ACL lock" );
 }
 
@@ -152,6 +168,13 @@ void server_unlock_acl( struct server *serve )
 }
 
 
+int server_acl_locked( struct server * serve )
+{
+	NULLCHECK( serve );
+	return flexthread_mutex_held( serve->l_acl );
+}
+
+
 /** Return the actual port the server bound to.  This is used because we
  * are allowed to pass "0" on the command-line.
  */
@@ -640,6 +663,11 @@ void serve_cleanup(struct server* params,
 			pthread_join(thread_id, &status);
 		}
 	}
+
+	if ( server_acl_locked( params ) ) {
+		server_unlock_acl( params );
+	}
+
 	debug( "Cleanup done");
 }