Free all possibly held mutexes in error handlers

Now that we have 3 mutexes lying around, it's important that we check
and free these if necessary if error() is called in any thread that can
hold them.  To do this, we now have flexthread.c, which defines a
flexthread_mutex struct.  This is a wrapper around a pthread_mutex_t and
a pthread_t.  The idea is that in the error handler, the thread can
check whether it holds the mutex and can free it if and only if it does.
This is important because pthread fast mutexes can be freed by *any*
thread, not just the thread which holds them.

Note: it is only ever safe for a thread to check if it holds the mutex
itself.  It is *never* safe to check if another thread holds a mutex
without first locking that mutex, which makes the whole operation rather
pointless.

tests/acceptance/fakes/dest/close_after_entrust.rb

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+# Open a server, accept a client, then we expect a single write
+# followed by an entrust. Disconnect after the entrust. We expect a
+# reconnection followed by a full mirror.
+
+require 'flexnbd/fake_dest'
+include FlexNBD
+
+addr, port, src_pid = *ARGV
+server = FakeDest.new( addr, port )
+client = server.accept
+
+client.write_hello
+write_req = client.read_request
+data = client.read_data( write_req[:len] )
+client.write_reply( write_req[:handle], 0 )
+
+entrust_req = client.read_request
+fail "Not an entrust" unless entrust_req[:type] == 65536
+client.close
+
+client2 = server.accept
+client2.receive_mirror
+
+exit(0)
+

tests/acceptance/flexnbd/fake_dest.rb

@@ -94,6 +94,31 @@ module FlexNBD
       end
 
 
+      def receive_mirror
+        write_hello()
+        loop do
+          req = read_request
+          case req[:type]
+          when 1
+            read_data( req[:len] )
+            write_reply( req[:handle] )
+          when 65536
+            write_reply( req[:handle] )
+            break
+          else
+            raise "Unexpected request: #{req.inspect}"
+          end
+        end
+
+        disc = read_request
+
+        if disc[:type] == 2
+          close
+        else
+          raise "Not a disconnect: #{req.inspect}"
+        end
+      end
+
     end # class Client
 
 

tests/acceptance/test_source_error_handling.rb

@@ -26,86 +26,72 @@ class TestSourceErrorHandling < Test::Unit::TestCase
 
 
   def test_destination_hangs_after_connect_reports_error_at_source
-    run_fake( "dest/hang_after_connect" )
-
-    stdout, stderr = @env.mirror12_unchecked
-    assert_match( /Remote server failed to respond/, stderr )
-    assert_success
+    run_fake( "dest/hang_after_connect",
+              /Remote server failed to respond/ )
   end
 
 
   def test_destination_rejects_connection_reports_error_at_source
-    run_fake( "dest/reject_acl" )
-
-    stdout, stderr = @env.mirror12_unchecked
-    assert_match /Mirror was rejected/, stderr
-    assert_success
+    run_fake( "dest/reject_acl",
+              /Mirror was rejected/ )
   end
 
   def test_wrong_size_causes_disconnect
-    run_fake( "dest/hello_wrong_size" )
-    stdout, stderr = @env.mirror12_unchecked
-    assert_match /Remote size does not match local size/, stderr
-    assert_success
+    run_fake( "dest/hello_wrong_size",
+              /Remote size does not match local size/ )
   end
 
 
   def test_wrong_magic_causes_disconnect
-    run_fake( "dest/hello_wrong_magic" )
-    stdout, stderr = @env.mirror12_unchecked
-    assert_match /Mirror was rejected/, stderr
-    assert_success "dest/hello_wrong_magic fake failed"
+    run_fake( "dest/hello_wrong_magic",
+              /Mirror was rejected/ )
   end
 
 
   def test_disconnect_after_hello_causes_retry
-    run_fake( "dest/close_after_hello" )
-    stdout, stderr = @env.mirror12_unchecked
-    assert_match( /Mirror started/, stdout )
-
-    assert_success
+    run_fake( "dest/close_after_hello",
+              /Mirror started/ )
   end
 
 
   def test_write_times_out_causes_retry
     run_fake( "dest/hang_after_write" )
-    stdout, stderr = @env.mirror12_unchecked
-
-    assert_success
   end
 
 
   def test_rejected_write_causes_retry
     run_fake( "dest/error_on_write" )
-    stdout, stderr = @env.mirror12_unchecked
-    assert_success
   end
 
 
   def test_disconnect_before_write_reply_causes_retry
     run_fake( "dest/close_after_write" )
-    @env.mirror12_unchecked
-    assert_success
   end
 
 
   def test_bad_write_reply_causes_retry
     run_fake( "dest/write_wrong_magic" )
-    @env.mirror12_unchecked
-    assert_success
   end
 
 
   def test_pre_entrust_disconnect_causes_retry
     run_fake( "dest/close_after_writes" )
-    @env.mirror12_unchecked
-    assert_success
+  end
+
+
+  def test_post_entrust_disconnect_causes_retry
+    @env.nbd1.can_die(0)
+    run_fake( "dest/close_after_entrust" )
   end
 
 
   private
-  def run_fake(name)
+  def run_fake(name, err=nil)
     @env.run_fake( name, @env.ip, @env.port2 )
+    stdout, stderr = @env.mirror12_unchecked
+    assert_success
+    assert_match( err, stderr ) if err
+    return stdout, stderr
   end
 
   def assert_success( msg=nil )