commit 1e46836b4646ae7847f8acd4e004550af9c0cedf Author: Nick Thomas Date: Sat Jun 4 22:53:33 2011 +0100 Initial import diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c111b33 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.gem diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..07c2496 --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,9 @@ +Changelog +========= + +0.1 +--- +Initial release + +In theory, parses all messages except for those related to transactions to IP +addresses. Some test coverage. diff --git a/README b/README new file mode 100644 index 0000000..df194fa --- /dev/null +++ b/README @@ -0,0 +1,18 @@ +README +====== + +btc_wire_proto is a library that serialises and deserialises messages in the +Bitcoin wire format. Getting the data from (and sending it to) the network, +finding peers, validating blocks, building a block chain, etc. are all out of +scope. + +Specification of the protocol is here: +https://en.bitcoin.it/wiki/Protocol_specification + +Alternative Ruby libraries that sort-of look at the same area: +https://github.com/altamic/bitcoin-protocol +https://github.com/lian/bitcoin-ruby + +This library differs from the others in that it leverages bindata to do the +heavy lifting. Results in less, easier-to-read, more robust code. + diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..e69de29 diff --git a/btc_wire_proto.gemspec b/btc_wire_proto.gemspec new file mode 100644 index 0000000..1cff54c --- /dev/null +++ b/btc_wire_proto.gemspec @@ -0,0 +1,19 @@ +Gem::Specification.new do |gem| + gem.name = "btc_wire_proto" + gem.version = "0.1" + gem.date = Date.today.to_s + gem.summary = "(De)serialiser for the Bitcoin p2p wire protocol" + gem.description = "Implements the generation and parsing code for Bitcoin " + + "p2p messages" + + gem.authors = ['Nick Thomas'] + gem.email = "nick@sharpcoin.org" + gem.homepage = "http://sharpcoin.org" + + gem.add_dependency('rake') + gem.add_dependency('ipaddress') + gem.add_dependency('bindata') + gem.add_development_dependency('rspec', ['>= 2.5.0']) + + gem.files = Dir['Rakefile', '{lib,spec}/**/*', 'README', 'CHANGELOG'] +end diff --git a/lib/btc_wire_proto.rb b/lib/btc_wire_proto.rb new file mode 100644 index 0000000..6e46130 --- /dev/null +++ b/lib/btc_wire_proto.rb @@ -0,0 +1,467 @@ +require 'bindata' +require 'ipaddress' + +# Wraps the IPAddress::IPv6 class found in the ipaddress gem to provide easier +# handling of binary-format IP addresses +# @author Nick Thomas +class Ipv6Address < BinData::Primitive + endian :big + + uint128 :address + + def get + if address + if address >= 0xffff00000000 && address <= 0xffffffffffff # v6-mapped v4 + IPAddress::IPv6::Mapped.parse_u128(address) + else # v6 + IPAddress::IPv6.parse_u128(address) + end + else + nil # Nothing set + end + end + + def set(v) + v = IPAddress::IPv6::Mapped.new(v.to_s) if v.is_a?(IPAddress::IPv4) + v = IPAddress(v.to_s) unless v.is_a?(IPAddress::IPv6) + + if v.respond_to?(:to_u128) + address = v.to_u128 + else + raise ArgumentError.new("Can't set #{v.class} to an IPv6Address") + end + end +end + + + +# Implementation of the BitCoin wire protocol, written using bindata. +# Reference: https://en.bitcoin.it/wiki/Protocol_specification +# +# @author Nick Thomas +module BtcWireProto + class << self + def CURRENT_VERSION(network = :main) + VERSIONS[network] + end + end + + # Comprehensive list of known networks. The hex values are what you see in + # MessageHdr#magic and the symbols are their known friendly names. + NETWORKS = { + :testnet => 0xDAB5BFFA, + 0xDAB5BFFA => :testnet, + + :main => 0xD9B4BEF9, + 0xD9B4BEF9 => :main + } + + # The current supported protocol version for the various networks. + VERSIONS = { + :main => 32100, + 32100 => :main + } + + # Comprehensive list of known inventory vector types. + INV_VEC_TYPES = { + 0 => :error, + :error => 0, + + 1 => :msg_tx, + :msg_tx => 1, + + 2 => :msg_block, + :msg_block => 2 + } + + # Used in Reply messages + REPLY_CODES = { + 0 => :success, + :success => 0, + + 1 => :wallet_error, + :wallet_error => 1, + + 2 => :denied, + :denied => 2 + } + + # Only Alert messages signed by this key are valid. + # This is an ECDSA public key (FIXME: in what format?) + ALERT_PUBKEY = "04fc9702847840aaf195de8442ebecedf5b095cdbb9bc716bda9110971b" + + "28a49e0ead8564ff0db22209e0374782c093bb899692d524e9d6a6956e7" + + "c5ecbcd68284" + + ## Components of payloads ## + + # Bitmask advertising various capabilities of the node. + # @author Nick Thomas + class ServicesMask < BinData::Record + endian :little + bit7 :top_undefined + bit1 :node_network + bit56 :undefined + end + + # Structure holding an IP address and port in a slightly unusual format. + # This one is big-endian - everything else is little-endian. + # + # @author Nick Thomas + class NetAddr < BinData::Record + endian :big + services_mask :services + ipv6_address :ip # IPv6 address. IPv4 addresses given as IPv6-mapped IPv4 + uint16 :port + end + + # Like a NetAddr but with a timestamp to boot. + # @author Nick Thomas + class TimestampedNetAddr < BinData::Record + endian :little + + uint32 :timestamp # TODO: Allow this to be set with Ruby native types + net_addr :net_addr + end + + # Variable-length integer. This is slightly scary. + # @author Nick Thomas + class VarInt < BinData::BasePrimitive + register_self + + def value_to_binary_string(val) + val = val.to_i + + if val < -0xffffffffffffffff # unrepresentable + "" + elsif val < 0 # 64-bit negative integer + top_32 = ((-val) & 0xffffffff00000000) >> 32 + btm_32 = (-val) & 0x00000000ffffffff + [0xff, top_32, btm_32].pack("CVV") + elsif val <= 0xfc # 8-bit (almost) positive integer + [val].pack("C") + elsif val <= 0xffff # 16-bit positive integer + [0xfd, val].pack("Cv") + elsif val <= 0xffffffff # 32-bit positive integer + [0xfe, val].pack("CV") + else # We can't represent this, whatever it is + "" + end + end + + def read_and_return_value(io) + magic = read_uint8(io) + if magic <= 0xfc # 8-bit (almost) positive integer + magic + elsif magic == 0xfd # 16-bit positive integer + read_uint16(io) + elsif magic == 0xfe # 32-bit positive integer + read_uint32(io) + elsif magic == 0xff # 64-bit negative integer + -(read_uint64(io)) + end + end + + def sensible_default + 0 + end + + protected + + def read_uint8(io) + io.readbytes(1).unpack("C").at(0) + end + + def read_uint16(io) + io.readbytes(2).unpack("v").at(0) + end + + def read_uint32(io) + io.readbytes(4).unpack("V").at(0) + end + + def read_uint64(io) + top, bottom = io.readbytes(8).unpack("VV") + (top << 32) | bottom + end + end + + # Variable-length pascal string with a variable-length int specifying the + # length. I kid you not. + # @author Nick Thomas + class VarStr < BinData::Primitive + endian :little + + var_int :len, :value => lambda { data.length } + string :data, :read_length => :len + + def get ; self.data ; end + def set(v) ; self.data = v ; end + end + + class InventoryVector < BinData::Record + endian :little + + uint32 :type # For values, see INV_VEC_TYPES + string :iv_hash, :length => 32 + end + + # Simple class wrapping raw SHA256 data. Might have utility methods later. + # @author Nick Thomas + class Sha256 < BinData::Record + string :data, :length => 32 # Raw SHA256 data + end + SHA256 = Sha256 + + # @author Nick Thomas + class TransactionIn < BinData::Record + endian :little + + sha256 :po_hash + uint32 :po_index + + var_str :signature_script # Script for confirming transaction authorisation + uint32 :sequence # Version of this record. + end + + # @author Nick Thomas + class TransactionOut < BinData::Record + endian :little + + uint64 :txout_value + var_str :pk_script # Script containing conditions to claim to transaction + end + + # Header for a block. + # @author Nick Thomas + class BlockHdr < BinData::Record + endian :little + + uint32 :version + sha256 :prev_block + sha256 :merkle_root + uint32 :timestamp + uint32 :difficulty + uint32 :nonce + var_int :txn_count + end + + ## Payloads ## + + # Payload for a version message + # @author Nick Thomas + class Version < BinData::Record + endian :little + + uint32 :version + services_mask :services + uint64 :timestamp + net_addr :addr_me + net_addr :addr_you, :onlyif => lambda { version >= 106 } + uint64 :nonce, :onlyif => lambda { version >= 106 } + var_str :sub_version, :onlyif => lambda { version >= 106 } + uint32 :start_height, :onlyif => lambda { version >= 209 } + end + + # Payload for an addr message in versions earlier than 31402. These are + # used to get a list of peers to interact with. + # @author Nick Thomas + class AddrPre31402 < BinData::Record + endian :little + + var_int :addr_count + array :addrs, :type => :net_addr, + :read_until => lambda { index == addr_count - 1 } + + end + + # Payload for an addr message in versions later than 31402. A timestamp was + # added to the list of addresses, but otherwise it's the same as AddrPre31402 + # @author Nick Thomas + class AddrFrom31402 < BinData::Record + endian :little + + var_int :addr_count + array :timestamped_addrs, :type => :timestamped_net_addr, + :read_until => lambda { index == addr_count - 1 } + end + + # Payload for a getdata or inv message. This lets the peer advertise the + # various objects it has knowledge of. + # @author Nick Thomas + class Inventory < BinData::Record + endian :little + + var_int :iv_count + array :items, :type => :inventory_vector, + :read_until => lambda { index == iv_count - 1 } + end + + # Payload for a getblocks or getheaders message. Specifies a set of blocks + # that the sender wants details of. + # @author Nick thomas + class BlockSpec < BinData::Record + endian :little + + uint32 :version + var_int :start_count + array :hash_start, :type => :sha256, + :read_until => lambda { index == start_count - 1 } + # Hash of the last desired block, or 0 to get as many as possible (max: 500) + sha256 :hash_stop, :length => 32 + end + + # A transaction. This contains a number of transactions 'in', and 'out'. + # @author Nick Thomas + class Transaction < BinData::Record + endian :little + + uint32 :version + var_int :tx_in_count + array :transactions_in, :type => :transaction_in, + :read_until => lambda { index == tx_in_count - 1 } + var_int :tx_out_count + array :transactions_out, :type => :transaction_out, + :read_until => lambda { index == tx_out_count - 1 } + uint32 :lock_time + end + + # Details about a particular block. Returned in response to a block request + # @author Nick Thomas + class Block < BinData::Record + endian :little + + block_hdr :header + array :txns, :type => :transaction, + :read_until => lambda { index == header.txn_count - 1 } + end + + # Headers payloads are returned in response to a getheaders request. + # Limit of 2,000 entries per message. + # @author Nick Thomas + class Headers < BinData::Record + endian :little + + var_int :hdr_count + array :block_hdrs, :type => :block_hdr, + :read_until => lambda { index == hdr_count - 1 } + + end + + # For now, we don't support CheckOrder requests at all. Protocol documentation + # is lacking! FIXME + # @author Nick Thomas + class CheckOrder < BinData::Record + endian :little + end + + # We don't support SubmitOrder replies either. Receiving either of these will + # actually break the stream, since we don't even know how long they are. FIXME + # @author Nick Thomas + class SubmitOrder < BinData::Record + endian :little + end + + # Used as a response to a CheckOrder request. + # @author Nick Thomas + class Reply < BinData::Record + endian :little + + uint32 :reply # See REPLYCODES for possible values + end + + # Completely empty payload. BinData dies if we don't specify *something* + # in the message payload choices. + # @author Nick Thomas + class NullPayload < BinData::Record + endian :little + end + + # A message sent using the p2p network. Signed by a key so you can tell who + # sent it - if it's signed by a particular key, then we should apparently + # show the message to the user and cease operation until further notice. Fun! + # @author Nick Thomas + class Alert < BinData::Record + endian :little + + var_str :message + var_str :signature + end + + ## Top-level message format ## + + # Found at the start of all Bitcoin messages. + # @author Nick Thomas + class MessageHdr < BinData::Record + endian :little + uint32 :magic + string :command, :length => 12 + uint32 :payload_len + uint32 :checksum, :onlyif => :has_checksum? + + protected + + # version and verack messages don't have a checksum. The rest do. + # @return[Boolean] does this message header have a checksum field or not? + def has_checksum? + !%w|version verack|.include?(command.strip) + end + end + + + # Everything on the wire is a Message. + # @author Nick Thomas + class Message < BinData::Record + + # @param[Fixnum,nil] version The protocol version. Setting this affects + # the layout of various fields. + def initialize_instance(v = nil) + super() + @version = v || ::BtcWireProto::CURRENT_VERSION(:main) + end + + message_hdr :header + + choice :payload, :selection => :payload_choice do + version "version" + addr_pre31402 "addr_pre31402" + addr_from31402 "addr_from31402" + inventory "inv" + inventory "getdata" + block_spec "getblocks" + block_spec "getheaders" + transaction "tx" + block "block" + headers "headers" + check_order "checkorder" + submit_order "submitorder" + alert "alert" + null_payload "null" + end + + # Works out what the payload looks like based on the MessageHdr struct + # and (potentially) the version + def payload_choice + cmd = header.command.to_s.strip + return cmd if %w{ + version inv getdata getblocks getheaders tx block headers alert + }.include?(cmd) + + # We can't parse these yet, and so we don't know where in the stream the + # next message starts. So all we can do is throw an error + raise NotImplementedError.new( + "Received unsupported command #{cmd}" + ) if %w|checkorder submitorder|.include?(cmd) + + # These commands don't have any payloads + return "null" if %w|verack getaddr ping|.include?(cmd) || cmd == "" + + # Payload has two forms, depending on protocol version. Ugh. + return (@version < 31402 ? "addr_pre31402" : "addr_from31402") if + cmd == "addr" + + raise NotImplementedError.new("Unknown command: #{cmd}") + end + + end +end + diff --git a/spec/btc_wire_proto_spec.rb b/spec/btc_wire_proto_spec.rb new file mode 100644 index 0000000..ca4b6ff --- /dev/null +++ b/spec/btc_wire_proto_spec.rb @@ -0,0 +1,282 @@ +require 'spec_helper' +require 'btc_wire_proto' + +# Fixtures data. Taken from: https://en.bitcoin.it/wiki/Protocol_specification + +include ::BtcWireProto + +describe ::BtcWireProto do + + # Payload fragments + describe ServicesMask do + it "should have node_network set to false when its bit is 0" do + s = ServicesMask::read("\x00" * 8) # All 64 bits unset + s.node_network.should == 0 + + s = ServicesMask::read(binary(%w{01 00 00 00 00 00 00 00})) + s.node_network.should == 1 + end + end + + describe NetAddr do + it "should have all fields set to 0 when the input data is all zeroes" do + na = NetAddr::read("\x00" * 26) + + na.services.node_network.should == 0 + na.ip.to_u128.should == 0 + na.port.should == 0 + end + + it "Should allow the Ip field to be set with Ruby native types" do + na = NetAddr::read("\x00" * 26) + mip = IPAddress("::ffff:0.0.0.1") + na.ip = mip + na.ip.to_u128.should == mip.to_u128 + end + + it "should have the fields set appropriately when fed binary data" do + na = NetAddr::read( + binary(%w{ + 01 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 FF FF 0A 00 00 01 20 8D + }) + ) + + na.services.node_network.should == 1 + na.ip.to_s.should == "::ffff:10.0.0.1" + na.port.should == 8333 + end + end + + + describe TimestampedNetAddr do + it "should leverage NetAddr" do + tna = TimestampedNetAddr::read("\x00" * 30) + tna.net_addr.class.should == BtcWireProto::NetAddr + end + + it "should have all fields set to 0 when the input data is all zeroes" do + tna = TimestampedNetAddr::read("\x00" * 30) + tna.timestamp.should == 0 + end + + it "should have the fields set appropriately when fed binary data" do + tna = TimestampedNetAddr::read( + binary(%w{ + E2 15 10 4D 01 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 FF FF 0A 00 00 01 20 8D + }) + ) + + tna.timestamp.should == 1292899810 + + tna.net_addr.services.node_network.should == 1 + tna.net_addr.ip.to_s.should == "::ffff:10.0.0.1" + tna.net_addr.port.should == 8333 + end + end + + + describe VarInt do + it "should hold numbers < 0x00000000 in nine bytes" do + bin_minus_1 = "\xff" + "\x00\x00\x00\x00" + "\x01\x00\x00\x00" + a = VarInt::read(bin_minus_1) + a.should == -1 + a.num_bytes.should == 9 + a.to_binary_s.should == bin_minus_1 + + a = VarInt::read("\xff" * 9) + a.should == -(2**64 - 1) + a.num_bytes.should == 9 + a.to_binary_s.should == "\xff" * 9 + end + + it "should hold numbers >= 0x00000000 and < 0x000000fd in one byte" do + a = VarInt::read("\x00") + a.should == 0x00 + a.num_bytes.should == 1 + a.to_binary_s.should == "\x00" + + a = VarInt::read("\xFC") + a.should == 0xFC + a.num_bytes.should == 1 + a.to_binary_s.should == "\xFC" + end + + it "should hold numbers >= 0x000000fd and < 0x00010000 in three bytes" do + a = VarInt::read("\xFD\xFD\x00") + a.should == 0xFD + a.num_bytes.should == 3 + a.to_binary_s.should == "\xFD\xFD\x00" + + a = VarInt::read("\xFD\xFF\xFF") + a.should == 0xFFFF + a.num_bytes.should == 3 + a.to_binary_s.should == "\xFD\xFF\xFF" + end + + it "should hold numbers >= 0x00010000 and < 0xffffffff in five bytes" do + a = VarInt::read("\xFE\x00\x00\x01\x00") + a.should == 0x10000 + a.num_bytes.should == 5 + a.to_binary_s.should == "\xFE\x00\x00\x01\x00" + + a = VarInt::read("\xFE\xFF\xFF\xFF\xFF") + a.should == 0xFFFFFFFF + a.num_bytes.should == 5 + a.to_binary_s.should == "\xFE\xFF\xFF\xFF\xFF" + + end + end + + describe VarStr do + it "should store string length in a var_int" do + a = VarStr::read("\x04abcd") + a.should == "abcd" + a.num_bytes.should == 5 + a.to_binary_s.should == "\x04abcd" + + a = VarStr::read("\xFD\xFF\xFF" + "A" * 0xFFFF) + a.should == "A" * 0xFFFF + a.num_bytes.should == 0xFFFF + 3 + a.to_binary_s.should == "\xFD\xFF\xFF" + "A" * 0xFFFF + end + end + + describe InventoryVector do + end + + describe Sha256 do + end + + describe TransactionIn do + end + + describe TransactionOut do + end + + describe BlockHdr do + end + + # Payloads + + describe Version do + it "should interpret binary data correctly" do + ver = Version::read(binary(%w{ + 9C 7C 00 00 01 00 00 00 00 00 00 00 E6 15 10 4D 00 00 00 00 + 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 FF FF + 0A 00 00 01 DA F6 01 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 FF FF 0A 00 00 02 20 8D DD 9D 20 2C 3A B4 57 13 + 00 55 81 01 00 + })) + + ver.version.should == 31900 + ver.services.node_network.should == 1 + ver.timestamp.should == 1292899814 + ver.addr_me.class.should == NetAddr + ver.addr_you.class.should == NetAddr + ver.nonce.should == 0x1357B43A2C209DDD + ver.sub_version.should == "" + ver.start_height.should == 98645 + end + + it "should exclude some fields by version" do + v = Version::read([1].pack("V") + "\x00" * 42) + v.num_bytes.should == 46 + v = Version::read([106].pack("V") + "\x00" * 77) + v.num_bytes.should == 81 + v = Version::read([209].pack("V") + "\x00" * 81) + v.num_bytes.should == 85 + end + end + + describe AddrPre31402 do + end + + describe AddrFrom31402 do + end + + describe Inventory do + end + + describe BlockSpec do + end + + describe Transaction do + end + + describe Block do + end + + describe Headers do + end + + describe CheckOrder do + end + + describe SubmitOrder do + end + + describe Reply do + end + + describe Alert do + end + + + # Messages + describe MessageHdr do + end + + describe Message do + context "Version message" do + it "should have a Version payload" do + m = Message::new(:header => {:command => 'version'}) + m.payload.selection.should == "version" + end + + it "should parse binary data correctly" do + m = Message::read(binary(%w{ + F9 BE B4 D9 76 65 72 73 69 6F 6E 00 00 00 00 00 55 00 00 + 00 9C 7C 00 00 01 00 00 00 00 00 00 00 E6 15 10 4D 00 00 + 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 FF FF 0A 00 00 01 DA F6 01 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 FF FF 0A 00 00 02 20 8D DD 9D 20 + 2C 3A B4 57 13 00 55 81 01 00 + })) + m.header.magic.should == BtcWireProto::NETWORKS[:main] + m.header.command.should == "version\x00\x00\x00\x00\x00" + m.header.payload_len.should == 85 + m.header.has_parameter?(:checksum).should be_false + m.payload.version.should == 31900 + m.payload.services.node_network.should == 1 + m.payload.timestamp.should == 1292899814 + m.payload.addr_me.class.should == NetAddr + m.payload.addr_you.class.should == NetAddr + m.payload.nonce.should == 0x1357B43A2C209DDD + m.payload.sub_version.should == "" + m.payload.start_height.should == 98645 + + end + end + + context "Verack message" do + it "should have no payload" do + m = Message.new(:header => {:command => "verack"}) + m.payload.selection.should == "null" + end + + it "should parse the binary data correctly" do + m = Message::read(binary(%w{ + F9 BE B4 D9 76 65 72 61 63 6B 00 00 00 00 00 00 00 00 00 00 + })) + m.header.magic.should == BtcWireProto::NETWORKS[:main] + m.header.command.should == "verack\x00\x00\x00\x00\x00\x00" + m.header.payload_len.should == 0 + m.header.has_parameter?(:chcksum).should be_false + end + + end + end + +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..217329f --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,10 @@ +require 'rspec' + +$: << File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib')) + +def binary(str_ary) + str_ary.collect do |d| + raise ArgumentError.new("Bad part") unless d =~ /\A[a-f0-9]{2}\Z/i + d.to_i(16).chr + end.join("") +end