Merge of doom

src/bitset.h

@@ -147,7 +147,8 @@ static inline int bitset_run_count(
 	uint64_t len)
 {
 	INT_FIRST_AND_LAST;
-	return bit_run_count(set->bits, first, bitlen) * set->resolution;
+	return (bit_run_count(set->bits, first, bitlen) * set->resolution) -
+	  (from % set->resolution);
 }
 
 /** Tests whether the bit field is clear for the given file offset.

src/client.c

@@ -61,7 +61,7 @@ void write_not_zeroes(struct client* client, off64_t from, int len)
 {
 	NULLCHECK( client );
 
-	char *map = client->serve->block_allocation_map;
+	struct bitset_mapping *map = client->serve->allocation_map;
 
 	while (len > 0) {
 		/* so we have to calculate how much of our input to consider
@@ -71,17 +71,31 @@ void write_not_zeroes(struct client* client, off64_t from, int len)
 		 * how many blocks our write covers, then cut off the start
 		 * and end to get the exact number of bytes.
 		 */
-		int first_bit = from/block_allocation_resolution;
-		int last_bit  = (from+len+block_allocation_resolution-1) / 
-		  block_allocation_resolution;
-		int run = bit_run_count(map, first_bit, last_bit-first_bit) *
-		  block_allocation_resolution;
 		 
-		if (run > len)
+		int run = bitset_run_count(map, from, len);
+		
+		debug("write_not_zeroes: from=%ld, len=%d, run=%d", from, len, run);
+		
+		if (run > len) {
 			run = len;
+			debug("(run adjusted to %d)", run);
+		}
 		
-		debug("write_not_zeroes: %ld+%d, first_bit=%d, last_bit=%d, run=%d", 
-		  from, len, first_bit, last_bit, run);
+		if (0) /* useful but expensive */
+		{
+			int i;
+			fprintf(stderr, "full map resolution=%d: ", map->resolution);
+			for (i=0; i<client->serve->size; i+=map->resolution) {
+				int here = (from >= i && from < i+map->resolution);
+				
+				if (here)
+					fprintf(stderr, ">");
+				fprintf(stderr, bitset_is_set_at(map, i) ? "1" : "0");
+				if (here)
+					fprintf(stderr, "<");
+			}
+			fprintf(stderr, "\n");
+		}
 		
 		#define DO_READ(dst, len) CLIENT_ERROR_ON_FAILURE( \
 			readloop( \
@@ -92,8 +106,8 @@ void write_not_zeroes(struct client* client, off64_t from, int len)
 			"read failed %ld+%d", from, (len) \
 		)
 		
-		if (bit_is_set(map, from/block_allocation_resolution)) {
-			debug("writing the lot");
+		if (bitset_is_set_at(map, from)) {
+			debug("writing the lot: from=%ld, run=%d", from, run);
 			/* already allocated, just write it all */
 			DO_READ(client->mapped + from, run);
 			server_dirty(client->serve, from, run);
@@ -104,16 +118,11 @@ void write_not_zeroes(struct client* client, off64_t from, int len)
 			char zerobuffer[block_allocation_resolution];
 			/* not allocated, read in block_allocation_resoution */
 			while (run > 0) {
-				char *dst = client->mapped+from;
-				int bit = from/block_allocation_resolution;
 				int blockrun = block_allocation_resolution - 
 				  (from % block_allocation_resolution);
 				if (blockrun > run)
 					blockrun = run;
 				
-				debug("writing partial: bit=%d, blockrun=%d (run=%d)",
-				  bit, blockrun, run);
-				
 				DO_READ(zerobuffer, blockrun);
 				
 				/* This reads the buffer twice in the worst case 
@@ -123,10 +132,10 @@ void write_not_zeroes(struct client* client, off64_t from, int len)
 				 */
 				if (zerobuffer[0] != 0 || 
 				    memcmp(zerobuffer, zerobuffer + 1, blockrun - 1)) {
-					memcpy(dst, zerobuffer, blockrun);
-					bit_set(map, bit);
+					debug("non-zero, writing from=%ld, blockrun=%d", from, blockrun);
+					memcpy(client->mapped+from, zerobuffer, blockrun);
+					bitset_set_range(map, from, blockrun);
 					server_dirty(client->serve, from, blockrun);
-					debug("non-zero, copied and set bit %d", bit);
 					/* at this point we could choose to
 					 * short-cut the rest of the write for
 					 * faster I/O but by continuing to do it
@@ -285,7 +294,7 @@ void client_reply_to_read( struct client* client, struct nbd_request request )
 void client_reply_to_write( struct client* client, struct nbd_request request )
 {
 	debug("request write %ld+%d", request.from, request.len);
-	if (client->serve->block_allocation_map) {
+	if (client->serve->allocation_map) {
 		write_not_zeroes( client, request.from, request.len );
 	}
 	else {

src/ioutil.c

@@ -14,10 +14,10 @@
 #include "util.h"
 #include "bitset.h"
 
-char* build_allocation_map(int fd, off64_t size, int resolution)
+struct bitset_mapping* build_allocation_map(int fd, off64_t size, int resolution)
 {
 	int i;
-	char *allocation_map = xmalloc((size+resolution)/resolution);
+	struct bitset_mapping* allocation_map = bitset_alloc(size, resolution);
 	struct fiemap *fiemap_count, *fiemap;
 
 	fiemap_count = (struct fiemap*) xmalloc(sizeof(struct fiemap));
@@ -49,27 +49,24 @@ char* build_allocation_map(int fd, off64_t size, int resolution)
 	if (ioctl(fd, FS_IOC_FIEMAP, fiemap) < 0)
 		return NULL;
 	
-	for (i=0;i<fiemap->fm_mapped_extents;i++) {
-		int first_bit = fiemap->fm_extents[i].fe_logical / resolution;
-		int last_bit  = (fiemap->fm_extents[i].fe_logical + 
-		  fiemap->fm_extents[i].fe_length + resolution - 1) / 
-		  resolution;
-		int run = last_bit - first_bit;
-		  
-		bit_set_range(allocation_map, first_bit, run);
-	}
+	for (i=0;i<fiemap->fm_mapped_extents;i++)
+		bitset_set_range(
+			allocation_map, 
+			fiemap->fm_extents[i].fe_logical, 
+			fiemap->fm_extents[i].fe_length
+		);
 	
 	for (i=0; i<16; i++) {
 		debug("map[%d] = %d%d%d%d%d%d%d%d",
 		  i,
-		  (allocation_map[i] & 1) == 1,
-		  (allocation_map[i] & 2) == 2,
-		  (allocation_map[i] & 4) == 4,
-		  (allocation_map[i] & 8) == 8,
-		  (allocation_map[i] & 16) == 16,
-		  (allocation_map[i] & 32) == 32,
-		  (allocation_map[i] & 64) == 64,
-		  (allocation_map[i] & 128) == 128
+		  (allocation_map->bits[i] & 1) == 1,
+		  (allocation_map->bits[i] & 2) == 2,
+		  (allocation_map->bits[i] & 4) == 4,
+		  (allocation_map->bits[i] & 8) == 8,
+		  (allocation_map->bits[i] & 16) == 16,
+		  (allocation_map->bits[i] & 32) == 32,
+		  (allocation_map->bits[i] & 64) == 64,
+		  (allocation_map->bits[i] & 128) == 128
 		);
 	}
 	

src/serve.c

@@ -431,7 +431,7 @@ void serve_init_allocation_map(struct server* params)
 	params->size = size;
 	SERVER_ERROR_ON_FAILURE(size, "Couldn't find size of %s", 
 	  params->filename);
-	params->block_allocation_map = 
+	params->allocation_map = 
 		build_allocation_map(fd, size, block_allocation_resolution);
 	close(fd);
 }
@@ -465,7 +465,7 @@ void serve_cleanup(struct server* params)
 
 	self_pipe_destroy( params->close_signal );
 
-	free(params->block_allocation_map);
+	free(params->allocation_map);
 	
 	if (params->mirror)
 		debug("mirror thread running! this should not happen!");

src/serve.h

@@ -76,7 +76,7 @@ struct server {
 	int                  server_fd;
 	int                  control_fd;
 	
-	char*                block_allocation_map;
+	struct bitset_mapping* allocation_map;
 	
 	struct client_tbl_entry nbd_client[MAX_NBD_CLIENTS];
 };

tests/flexnbd.rb

@@ -99,7 +99,8 @@ class FlexNBD
     IO.popen(cmd) do |fh|
       return fh.read
     end
-    raise "read failed" unless $?.success?
+    raise IOError.new "NBD read failed" unless $?.success?
+    out
   end
 
   def write(offset, data)
@@ -109,7 +110,7 @@ class FlexNBD
     IO.popen(cmd, "w") do |fh|
       fh.write(data)
     end
-    raise "write failed" unless $?.success?
+    raise IOError.new "NBD write failed" unless $?.success?
     nil
   end
 

tests/fuzz

@@ -22,7 +22,7 @@ end
 testname_local = "#{$0}.test.#{$$}.local"
 testname_serve = "#{$0}.test.#{$$}.serve"
 [testname_local, testname_serve].each do |name|
-  File.open(name, "w+") { |fh| fh.seek(test_size-1, IO::SEEK_SET); fh.write("0") }
+  File.open(name, "w+") { |fh| fh.seek(test_size-1, IO::SEEK_SET); fh.write("\0") }
 end
 
 @local = File.open(testname_local, "r+")
@@ -30,6 +30,13 @@ end
 @serve = FlexNBD.new(binary, "127.0.0.1", 41234)
 @serve.serve(testname_serve)
 
+$record = []
+def print_record
+  $record.each do |offset, length, byte|
+    STDERR.print "  wrote #{byte} to #{offset}+#{length}\n"
+  end
+end
+
 repetitions.times do |n|
 
   begin
@@ -44,6 +51,7 @@ repetitions.times do |n|
   
     if md5_local != md5_serve
       STDERR.print "Before pass #{n}: MD5 error: local=#{md5_local} serve=#{md5_serve}\n"
+      print_record
       STDERR.print "**** Local contents:\n"
       system("hexdump #{testname_local}")
       STDERR.print "**** Serve contents:\n"
@@ -51,24 +59,35 @@ repetitions.times do |n|
       exit 1
     end
 
-    length = rand(max_length)
+    length = rand(max_length/8)
     length &= 0xfffff000 if CHEAT_AND_ROUND_DOWN
     offset = rand(test_size - length)
     offset &= 0xfffff000 if CHEAT_AND_ROUND_DOWN
-    content = (n%2 == 1) ? ("x" * length) : ("\0" * length)
+    
+    content = (n%2 == 0) ? ("\0" * length) : ( (n&255).chr * length)
+    
+    $record << [offset, length, content[0]]
         
     @local.seek(offset, IO::SEEK_SET)
     @local.write(content)
+    @local.fsync
   
     @serve.write(offset, content)
   
-    if @serve.read(offset, length) != content
-      STDERR.print "After pass #{n}: Didn't read back what we wrote!"
+    check_read = @serve.read(offset, length)
+    if check_read != content
+      STDERR.print "After pass #{n}: Didn't read back what we wrote!\n"
+      print_record
+      STDERR.print "*** We wrote these #{content.length} bytes...\n"
+      IO.popen("hexdump", "w") { |io| io.print(content) }
+      STDERR.print "*** But we got back these #{check_read.length} bytes...\n"
+      IO.popen("hexdump", "w") { |io| io.print(check_read) }
       exit 1
     end
     
   rescue StandardError => ex
     STDERR.print "During pass #{n}: Exception: #{ex}"
+      print_record
     STDERR.print ex.backtrace.join("\n") + "\n"
     exit 2
   end