From c4598590c2ffd7d10dfd8b9febf7dc0fba9e2584 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Wed, 21 Mar 2018 05:08:24 +0000 Subject: [PATCH] Another night of .obj failure --- cmd/view-set/main.go | 45 ++++++++----- doc/formats/obj.md | 104 ++++++++++++++++++++++-------- internal/data/object.go | 109 ++++++++++++++----------------- scripts/try-uncompress | 138 +++++++++++++++++++++++++++++----------- 4 files changed, 254 insertions(+), 142 deletions(-) diff --git a/cmd/view-set/main.go b/cmd/view-set/main.go index 0ce7211..610dd86 100644 --- a/cmd/view-set/main.go +++ b/cmd/view-set/main.go @@ -2,8 +2,9 @@ package main import ( "flag" - "image/color" +// "image/color" "log" + "math" "os" "path/filepath" @@ -30,6 +31,7 @@ type env struct { type state struct { env *env + step int objIdx int spriteIdx int @@ -89,10 +91,17 @@ func (e *env) run() { oldState := *state state = state.runStep(pWin) - if oldState != *state { + if oldState != *state || oldState.step == 0 { log.Printf( - "new state: numObj=%d object=%d (%s) numFrames=%d frame=%d", // FIXME: rename to sprite throughout - state.env.set.Count(), state.objIdx, state.env.set.Palette[state.objIdx], state.curObject().NumFrames, state.spriteIdx) + "new state: numObj=%d object=%d (%s) numFrames=%d sprite=%d zoom=%.2f", + state.env.set.Count(), + state.objIdx, + state.env.set.Palette[state.objIdx], + state.curObject().NumSprites, + state.spriteIdx, + state.zoom, + ) + state.step += 1 state.present(pWin) } @@ -107,22 +116,21 @@ func (s *state) runStep(pWin *pixelgl.Window) *state { } func (s *state) present(pWin *pixelgl.Window) { - obj := s.curObject() - frame := obj.Frames[s.spriteIdx] // FIXME: Rename Frame to Sprite throughout +// obj := s.curObject() +// sprite := obj.Sprites[s.spriteIdx] - log.Printf("%#v", frame) + center := pWin.Bounds().Center() - pic := pixel.MakePictureData(pixel.R(0, 0, float64(frame.Width), float64(frame.Height))) - - // FIXME: how do I convert? Do I even have the right data here? - for i, b := range frame.PixelData { - pic.Pix[i] = color.RGBA{b, b, b, 255} - } - - sprite := pixel.NewSprite(pic, pic.Bounds()) + cam := pixel.IM +// cam = cam.ScaledXY(pixel.ZV, pixel.Vec{1.0, -1.0}) // invert the Y axis + cam = cam.Scaled(center, s.zoom) // apply current zoom factor + //cam = cam.Moved(center.Sub(s.camPos)) // Make it central + //cam = cam.Rotated(center, -0.785) // Apply isometric angle + s.cam = cam + pWin.SetMatrix(s.cam) pWin.Clear(colornames.White) - sprite.Draw(pWin, pixel.IM.Moved(pWin.Bounds().Center())) +// pixel.NewSprite(pic, pic.Bounds()).Draw(pWin, pixel.IM.Moved(center)) } func (s *state) handleKeys(pWin *pixelgl.Window) { @@ -147,10 +155,13 @@ func (s *state) handleKeys(pWin *pixelgl.Window) { } if pWin.JustPressed(pixelgl.KeyUp) { - if s.spriteIdx < int(s.curObject().NumFrames)-1 { + if s.spriteIdx < int(s.curObject().NumSprites)-1 { s.spriteIdx += 1 } } + + // Zoom in and out with the mouse wheel + s.zoom *= math.Pow(1.2, pWin.MouseScroll().Y) } func (s *state) curObject() *data.Object { diff --git a/doc/formats/obj.md b/doc/formats/obj.md index 4a8bbff..a35e9df 100644 --- a/doc/formats/obj.md +++ b/doc/formats/obj.md @@ -142,12 +142,16 @@ I think this is rendered as a 1px dot with the colour `#ff00ff` in WH40K_TD.exe: ![blank.obj as rendered by WH40K_TD.exe](img/blank.obj.png) -In the 64x64 tile, the dot is in the very centre. +In the tile, the dot is in the very centre. The colour itself doesn't show up in the data directly, so it's not a simple RGB -array of pixels. WH40K_TD.exe is a 16-bit-colour application and may use a -palette, but these factoids don't help me make immediate sense of the data. If -there's a palette in the first 24 bytes, it's not obvious. +array of pixels. WH40K_TD.exe is a 256-colour application, so the pixel data may +account for 1 of the 3 bytes. + +All sprites seem to end with 0x00 - like a GIF image descriptor, this may say +"end of data". + +So what's the 1? Simple RLE, like FLIC type 15 BYTE_RUN? - "Repeat 253 one time"? There are 45 `TYPE 13` .obj files in the game. Comparing the above with the start of the second sprite in `pillar.obj`: @@ -177,46 +181,94 @@ start of the second sprite in `pillar.obj`: Total size is 3340. The first 24 bytes look quite different to the remainder of this file, lending weight to the 24-byte header theory. -Line-by-line comparison: +Line-by-line comparisons of first 16 bytes (all TYPE 13). 0x10..0x17 are all +0x00 in all examples so far. + +``` +a: blank.obj sprite 0, 1x1 tile (27 bytes) +b: pillar.obj sprite 1 (3340 bytes) +c: altar.obj sprite 0 (7465 bytes) +d: altar.obj sprite 1 (7368 bytes) a 0x0000 10 01 61 01 01 00 01 00 | a | 16 1 97 1 1 0 1 0 b 0x0000 fa 00 25 01 2e 00 48 00 | H | 250 0 37 1 46 0 72 0 +c 0x0000 d6 00 16 01 74 00 67 00 | t g | 214 0 22 1 116 0 103 0 +d 0x0000 d6 00 19 01 74 00 64 00 | t d | 214 0 25 1 116 0 100 0 + a 0x0008 00 00 00 00 03 00 00 00 | | 0 0 0 0 3 0 0 0 b 0x0008 00 00 00 00 f4 0c 00 00 | | 0 0 0 0 244 12 0 0 +c 0x0008 00 00 00 00 11 1d 00 00 | | 0 0 0 0 17 29 0 0 +d 0x0008 00 00 00 00 b0 1c 00 00 | | 0 0 0 0 176 28 0 0 +``` -a 0x0010 00 00 00 00 00 00 00 00 | | 0 0 0 0 0 0 0 0 -b 0x0010 00 00 00 00 00 00 00 00 | | 0 0 0 0 0 0 0 0 +Assuming a 24-byte header, 0x0c matches "remaining pixeldata" size in all cases. -Assuming a 24-byte header, 0x0c matches "remaining pixeldata" size in both cases -- 3 and 3316. +0x04 makes sense as x,y dimension in `blank.obj` where we *know* it's 1x1 pixel. -It's odd that the last 8 bytes of this putative header are empty in both cases. -I need to see if I can find any examples that store it. - -0x04 makes sense as height & width, 2 16-bit values, in `blank.obj` where we -*know* it's 1x1 pixel. - -It makes less sense as such for `pillar.obj` where the individual sprites fit -into the 64x64x64 volume of a cell: - -![pillar.obj as individual sprites](img/pillar_6_sprites.png) +The pillar has more Y than X; the altar more X than Y, suggesting 0x04-0x05 are +X and 0x06-0x07 are Y. Measuring the rendered pixels by WH40K_TD.exe gives a +close correspondence in all cases. WH40K_TD.exe crashes trying to load a set referencing `pillar` in it unless it's in the CENTER position. Interesting. -Next: keep adding cases to the header comparison above until I make sense of it -all. Since `pillar.obj` is composable, it's not the best example I could have -picked. Need more non-composable ones, in case that makes a difference. +Not all pixeldatas are evenly divisible by 3. blank.obj seems to be the special +case. +The volume represented by a cell is a little odd. We see three faces of a fake +3D volume of size 64x64x32(ish). This is presented in an isomorphic fashion, so +the height is 32px at the leftmost and rightmost extents and 96 in the centre. +Overall width is 128px, and the minimum rectangle covering the whole space would +be 128x96, or 12288 bytes at 8bpp. + +Or it could be represented as the three planes separately. So for an altar, we'd +have 64x64 pixels, plus 32x64 pixels, plus 32x64 pixels (minus a few around the +edges, perhaps) - 4096 + 2048 + 2048 = 8192. The sprite is only slightly smaller +than that. + +Or perhaps we draw a normal X/Y rectangle which is then skewed appropriately. +That seems like it would be odd for pixeldata though? + +Bytes per pixel (intuited from hypothetical x,y) for the four frames so far: + +``` +a: 1 * 1 = 1. 27 - 24 = 3. 24 bits / pixel +b: 46 * 72 = 3312. 3340 - 24 = 3316. 8 bits / pixel with 4 bytes left over. +c: 116 * 103 = 11948. 7465 - 24 = 7439. 4.9 bits / pixel +d: 116 * 100 = 11600. 7368 - 24 = 7344. 5.1 bits / pixel +``` | Offset | Purpose | | ------ | ------- | | 0x0000 | ? -| 0x0004 | ? -| 0x0008 | ? +| 0x0004 | x,y size (16 bits each) | +| 0x0008 | ? (blank in all cases so far) | 0x000c | Size of remaining pixeldata | -| 0x0010 | Padding? -| 0x0014 | Padding? +| 0x0010 | Padding? | +| 0x0014 | Padding? | +We still don't know what the first 32 bits are all about. Perhaps they can help +to explain the differences in putative bpp. + +0x002-0x003 changes in step with total number of pixels? + +| ID | 0x2-0x3 | dec LE | Number of pixels | +| -- | ------- | ------ | ---------------- | +| a | 61 01 | 353 | 1 | +| b | 25 01 | 293 | 3312 | +| d | 19 01 | 281 | 11600 | +| c | 16 01 | 278 | 11948 | + +Not clear what it means. If anything! + +WH40K_TD.exe loops around "ReadInMissionFLCs", incl. address 0x0041dd10, where +it loads in all the .asc and .obj files in a set. + +``` +break *0x41DD10 +``` + +This lets me focus very narrowly on what happens when loading sprites, and +might give clues. diff --git a/internal/data/object.go b/internal/data/object.go index 45cbdd0..d226688 100644 --- a/internal/data/object.go +++ b/internal/data/object.go @@ -67,85 +67,71 @@ var ( "j_tree2.obj", // ObjectHeader is completely empty "inven.obj", // Main header padding contains unknown values: [134744072 134744072 134744072] } - objFrameMagic = uint32(0x014200D1) ) func init() { sort.Strings(objBlacklist) } -type FrameHeader struct { - Magic uint32 - Width uint16 // FIXME: I'm not certain this is what these are. If they are, they may be the wrong way around - Height uint16 +type SpriteHeader struct { + Unknown0 uint32 + Width uint16 // FIXME: I'm not certain this is what these are. + Height uint16 // FIXME: If they are, they may be the wrong way around Padding1 uint32 // I don't think this is used. Could be wrong. - PixelSize uint32 // Size of PixelData, excluding this frame header + PixelSize uint32 // Size of PixelData, excluding this sprite header Padding2 uint64 // I don't think this is used either. Could be wrong. } -func (f FrameHeader) Check(expectedSize uint32) error { - // There seem to be different frame types, keyed by the magic value? - // if f.Magic != objFrameMagic { - // return fmt.Errorf("Unexpected magic value: %d (expected %d)", f.Magic, objFrameMagic) - // } - - if f.Padding1 != 0 || f.Padding2 != 0 { - return fmt.Errorf("Frame header padding contains unknown values: %d %d", f.Padding1, f.Padding2) +func (s SpriteHeader) Check(expectedSize uint32) error { + if s.Padding1 != 0 || s.Padding2 != 0 { + return fmt.Errorf("Sprite header padding contains unknown values: %d %d", s.Padding1, s.Padding2) } // Remove 24 bytes from passed-in size to account for the header - if f.PixelSize != expectedSize-24 { - return fmt.Errorf("Advertised pixel size: %d differs from expected: %v", f.PixelSize, expectedSize-24) + if s.PixelSize != expectedSize-24 { + return fmt.Errorf("Advertised pixel size: %d differs from expected: %v", s.PixelSize, expectedSize-24) } return nil } -type Frame struct { - FrameHeader +type Sprite struct { + SpriteHeader PixelData []byte } -type frameInfoHeader struct { - Offset uint32 // Offset of the frame relative to the frame data segment - Size uint32 // Size of the frame in bytes, including the header +type dirEntry struct { + Offset uint32 // Offset of the sprite relative to the data block + Size uint32 // Size of the sprite in bytes, including any header } -func (f frameInfoHeader) Check() error { - if f.Size < 24 { - return fmt.Errorf("Unexpected frame size: %d (expected >= 24)", f.Size) +func (d dirEntry) Check() error { + if d.Size < 24 { + return fmt.Errorf("Unexpected sprite size: %d (expected >= 24)", d.Size) } return nil } -// ObjectHeader totals 32 bytes on disk +// ObjectHeader totals 24 bytes on disk type ObjectHeader struct { - NumFrames uint32 // How many frames does this object have? - MainHeaderSize uint32 // Number of bytes taken by this header. Should always be `32` - FrameInfoSize uint32 // Number of bytes taken up by the next block. 8 * NumFrames - FrameDataOffset uint32 // The starting point of the frame data - FrameDataSize uint32 // frame data should take up this many bytes - - Padding [3]uint32 // Unused, as far as I can see + NumSprites uint32 // How many sprites does this object have? + DirOffset uint32 // Offset of the directory block + DirSize uint32 // Size of the directory block. 8 * NumSprites + DataOffset uint32 // Offset of the sprite data block + DataSize uint32 // Size of the sprite data block } -func (h ObjectHeader) ExpectedFrameInfoSize() uint32 { - return h.NumFrames * 8 +func (h ObjectHeader) ExpectedDirSize() uint32 { + return h.NumSprites * 8 } func (h ObjectHeader) Check() error { - if h.MainHeaderSize != 32 { - return fmt.Errorf("Unexpected main header size: %d (expected 32)", h.MainHeaderSize) - } + // TODO: check for overlaps - if h.ExpectedFrameInfoSize() != h.FrameInfoSize { - return fmt.Errorf("Unexpected frame info size: %d (expected %d)", h.FrameInfoSize, h.ExpectedFrameInfoSize()) - } - - if h.Padding[0] != 0 || h.Padding[1] != 0 || h.Padding[2] != 0 { - return fmt.Errorf("Main header padding contains unknown values: %+v", h.Padding) + if h.ExpectedDirSize() != h.DirSize { + return fmt.Errorf("Unexpected sprite directory size: %d (expected %d)", h.DirSize, h.ExpectedDirSize()) } return nil @@ -155,7 +141,7 @@ type Object struct { ObjectHeader Filename string - Frames []*Frame + Sprites []*Sprite } func LoadObject(filename string) (*Object, error) { @@ -175,14 +161,17 @@ func LoadObject(filename string) (*Object, error) { return nil, err } - // Now load all frames into memory - - framesInfo := make([]frameInfoHeader, out.NumFrames) - if err := binary.Read(f, binary.LittleEndian, &framesInfo); err != nil { + // Now load all sprites into memory + dir := make([]dirEntry, out.NumSprites) + if _, err := f.Seek(int64(out.DirOffset), io.SeekStart); err != nil { return nil, err } - if _, err := f.Seek(int64(out.FrameDataOffset), io.SeekStart); err != nil { + if err := binary.Read(f, binary.LittleEndian, &dir); err != nil { + return nil, err + } + + if _, err := f.Seek(int64(out.DataOffset), io.SeekStart); err != nil { return nil, err } @@ -195,42 +184,38 @@ func LoadObject(filename string) (*Object, error) { buf := bytes.NewReader(data) - for _, frameInfo := range framesInfo { - if err := frameInfo.Check(); err != nil { + for _, dirEntry := range dir { + if err := dirEntry.Check(); err != nil { return nil, err } - if _, err := buf.Seek(int64(frameInfo.Offset), io.SeekStart); err != nil { + if _, err := buf.Seek(int64(dirEntry.Offset), io.SeekStart); err != nil { return nil, err } - frame := &Frame{} + sprite := &Sprite{} - if err := binary.Read(buf, binary.LittleEndian, &frame.FrameHeader); err != nil { + if err := binary.Read(buf, binary.LittleEndian, &sprite.SpriteHeader); err != nil { return nil, err } - if err := frame.Check(frameInfo.Size); err != nil { + if err := sprite.Check(dirEntry.Size); err != nil { return nil, err } // It's safe to assume that a `bytes.Reader` will always satisfy the // requested read size. - frame.PixelData = make([]byte, frame.PixelSize) - if _, err := buf.Read(frame.PixelData); err != nil { + sprite.PixelData = make([]byte, sprite.PixelSize) + if _, err := buf.Read(sprite.PixelData); err != nil { return nil, err } - out.Frames = append(out.Frames, frame) + out.Sprites = append(out.Sprites, sprite) } return out, nil } -func readObjFrame(f io.Reader, obj *Object) error { - return nil -} - func LoadObjects(dir string) (map[string]*Object, error) { fis, err := ioutil.ReadDir(dir) if err != nil { diff --git a/scripts/try-uncompress b/scripts/try-uncompress index f82dd3f..e24de5a 100755 --- a/scripts/try-uncompress +++ b/scripts/try-uncompress @@ -19,7 +19,6 @@ module Obj def self.parse(data) hdr = new(*data[0..SIZE - 1].unpack("V*")) - pp hdr hdr.validate!(data.bytes.size) hdr end @@ -82,8 +81,6 @@ module Obj DirEntry.parse(rel_data.byteslice(rel_offset, DirEntry::SIZE)) end - pp entries - new(entries) end @@ -94,38 +91,58 @@ module Obj # Convert the directory into an Array of bytes. Until we work out how to # parse sprites, anyway... def realize(rel_data) - entries.map { |entry| rel_data[entry.sprite_range] } + entries.map { |entry| Sprite.parse(rel_data[entry.sprite_range]) } end end -=begin - SpriteHeader = Struct.new( - :unknown0, # Possibly magic data? It's the same for every sprite in jungtil.obj - :maybe_dimension, # Low nibble comes to 63 in jungtil.obj which would work for a 64x64 tile - :size, # Number of bytes of pixel data following this header - ) do - SIZE = 4*6 # Seems to be, anyway. Based on + class SpriteHeader + SIZE = 24 # Seems to be, anyway. Based on the size here vs. the size in the dir + + attr_reader( + :unknown0, # Possibly magic data? It's the same for every sprite in jungtil.obj + :width, # Low nibble comes to 63 in jungtil.obj which would work for a 64x64 tile + :height, + :unknown8, + :size, # Number of bytes of pixel data following this header + :unknown16, + :unknown20, + ) def self.parse(rel_data) - new(*sprite_data[0..SIZE-1]).unpack("V*") + new(*rel_data[0...SIZE].unpack("VvvVVVV")) + end + + def initialize(*args) + @unknown0, + @width, + @height, + @unknown8, + @size, + @unknown16, + @unknown20 = *args end def pixel_range - SIZE...size # maybe,anyway + SIZE...(SIZE+size) end end - Sprite = Struct.new( - :header, :data - ) do + class Sprite + attr_reader :header, :data, :raw + def self.parse(rel_data) hdr = SpriteHeader.parse(rel_data) sprite_pixels = rel_data[hdr.pixel_range] - Sprite.new(hdr, sprite_pixels) + new(hdr, sprite_pixels, rel_data) + end + + def initialize(header, data, raw = nil) + @header = header + @data = data + @raw = raw end end -=end class Parsed attr_reader :header @@ -210,10 +227,11 @@ end def dump(filename, spriteno = -1) obj = load_obj(filename) +# pp obj if spriteno == -1 puts "Dumping all sprites for #{filename}" - obj.sprites.each { |sprite| display(sprite, header: true) } + obj.sprites.each { |sprite| display(sprite.raw, 2, header: true) } else puts "Dumping sprite #{spriteno} for #{filename}" display(obj.sprites[spriteno]) @@ -227,37 +245,61 @@ end # * DEFLATE / ZLIB # * LZO # * LZOP +# * RLE (maybe?) +# +# Maybe try: +# * RLE8: https://www.fileformat.info/format/bmp/corion-rle8.htm def decompress(filename) - puts "\nAttempting decompression of #{filename}..." - - require 'zlib' + puts "\nAttempting RLE decompression of #{filename}..." obj = load_obj(filename) obj.sprites.each_with_index do |sprite, i| print "Sprite %02d..."%i - # Step through the start of each sprite so we aren't stopped by a hypothetical header - (0...64).each do |offset| - block = sprite.byteslice(offset, sprite.size-offset) + decompressed = [] + data = sprite.data.bytes + hdr = sprite.header - begin - inflater = Zlib::Inflate.new(-32) - decompressed = "" - decompressed << inflater.inflate(block) - raise "0 bytes" if decompressed.size == 0 + (0...hdr.height).each do |i| + # npackets = data.shift(1)[0] # ignore the packet byte + decompressed_line = [] - puts "succeeded! sprite=#{i} offset=#{offset} decompressed_size=#{decompressed.size}" - puts "data:" - puts decompressed.inspect - exit 0 - rescue => err - puts "failed: #{err}" + count = 0 + loop do + cmd = data.shift(1)[0] + if cmd == 0 # end of stream + print "Done! #{data.size} bytes left. " + break + elsif cmd == nil + print "Ran out of data! " + break + elsif cmd > 128 # "negative" bytes say "copy abs(X) bytes unmodified from input to output" + decompressed_line.concat(data.shift(cmd-128)) + else # "positive" bytes say "repeat the next byte X times" + decompressed_line.concat(data.shift(1)*cmd) + end + + if decompressed_line.size == hdr.width + print "Done line! " + break + end + + # if npackets > 0 && count == npackets + # print "Done packets! " + # break + # end + + count+=1 end + + raise "Bad length for line #{i}! #{decompressed_line.size} (Expected #{hdr.width})" if decompressed_line.size != hdr.width + decompressed << decompressed_line end - #puts "failed" + puts "Decompressed: #{decompressed.flatten.size} bytes. Sprite pixels: #{hdr.width*hdr.height}" + #display(decompressed.map(&:chr).join("")) end end @@ -277,7 +319,29 @@ def compare(filenames) end end +def sprites(filename) + obj = load_obj(filename) + + puts filename + ":" + puts "\ti X Y Px bytes bits/px" + obj.sprites.each_with_index do |spr, i| + hdr = spr.header + px = hdr.width * hdr.height + + puts "\t%02d: %03d %03d %04d %04d %.2f"%[ + i, + hdr.width, + hdr.height, + px, + hdr.size, + (hdr.size*8) / px.to_f + ] + end +end + case command = ARGV.shift +when "sprites" then + ARGV.each { |filename| sprites(filename) } when "dump" then ARGV.each { |filename| dump(filename) } when "compare" then