More .obj investigating. 0x80 seems to be a special value

.gitignore

@@ -1,6 +1,7 @@
 /investigation
 /loader
 /orig
+/view-obj
 /view-map
 /view-minimap
 /view-set

Makefile

@@ -1,10 +1,12 @@
 srcfiles = $(shell find . -iname *.go)
 
-all: loader view-map view-minimap view-set
+all: loader view-obj view-map view-minimap view-set
 
 loader: $(srcfiles)
 	go build -o loader ur.gs/ordoor/cmd/loader
 
+view-obj: $(srcfiles)
+	go build -o view-obj ur.gs/ordoor/cmd/view-obj
 
 view-map: $(srcfiles)
 	go build -o view-map ur.gs/ordoor/cmd/view-map
@@ -16,6 +18,6 @@ view-set: $(srcfiles)
 	go build -o view-set ur.gs/ordoor/cmd/view-set
 
 clean:
-	rm -f loader view-map view-minimap view-set
+	rm -f loader view-obj view-map view-minimap view-set
 
 .PHONY: all clean

cmd/view-map/main.go

@@ -158,8 +158,8 @@ func (s *state) present(pWin *pixelgl.Window) {
 
 	// TODO: bounds clipping
 	z := int(s.zIdx)
-	for y := int(gameMap.MaxLength - 1); y >= int(gameMap.MinLength); y-- {
+	for y := int(gameMap.MinLength); y < int(gameMap.MaxLength); y++ {
-		for x := int(gameMap.MaxWidth - 1); x >= int(gameMap.MinWidth); x-- {
+		for x := int(gameMap.MinWidth); x < int(gameMap.MaxWidth); x++ {
 
 			cell := gameMap.Cells.At(x, y, z)
 

cmd/view-obj/main.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"math"
+	"os"
+	"path/filepath"
+
+	"github.com/faiface/pixel"
+	"github.com/faiface/pixel/pixelgl"
+	"golang.org/x/image/colornames"
+
+	"ur.gs/ordoor/internal/conv"
+	"ur.gs/ordoor/internal/data"
+	"ur.gs/ordoor/internal/ui"
+)
+
+var (
+	gamePath = flag.String("game-path", "./orig", "Path to a WH40K: Chaos Gate installation")
+	objFile  = flag.String("obj", "", "Path to a .obj file, e.g. ./orig/Obj/TZEENTCH.OBJ")
+)
+
+type env struct {
+	obj *conv.Object
+}
+
+type state struct {
+	env *env
+
+	step      int
+	spriteIdx int
+
+	zoom float64
+
+	cam    pixel.Matrix
+	camPos pixel.Vec
+}
+
+func main() {
+	flag.Parse()
+
+	if *gamePath == "" || *objFile == "" {
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	rawObj, err := data.LoadObject(*objFile)
+	if err != nil {
+		log.Fatalf("Failed to load %s: %v", *objFile, err)
+	}
+	obj := conv.ConvertObject(rawObj, filepath.Base(*objFile))
+
+	env := &env{obj: obj}
+
+	// The main thread now belongs to pixelgl
+	pixelgl.Run(env.run)
+}
+
+func (e *env) run() {
+	win, err := ui.NewWindow("View Object: " + *objFile)
+	if err != nil {
+		log.Fatal("Couldn't create window: %v", err)
+	}
+
+	pWin := win.PixelWindow
+	state := &state{
+		env:    e,
+		camPos: pixel.V(0, float64(-pWin.Bounds().Size().Y)),
+		zoom:   8.0,
+	}
+
+	// For now, just try to display the various objects
+	// left + right to change object, up + down to change frame
+	win.Run(func() {
+		oldState := *state
+		state = state.runStep(pWin)
+
+		if oldState != *state || oldState.step == 0 {
+			log.Printf(
+				"new state: numSprites=%d sprite=%d zoom=%.2f",
+				len(state.env.obj.Sprites),
+				state.spriteIdx,
+				state.zoom,
+			)
+			state.present(pWin)
+		}
+
+		state.step += 1
+	})
+}
+
+func (s *state) runStep(pWin *pixelgl.Window) *state {
+	newState := *s
+	newState.handleKeys(pWin)
+
+	return &newState
+}
+
+func (s *state) present(pWin *pixelgl.Window) {
+	obj := s.env.obj
+	sprite := obj.Sprites[s.spriteIdx]
+
+	center := pWin.Bounds().Center()
+
+	cam := pixel.IM
+	cam = cam.ScaledXY(center, pixel.Vec{1.0, -1.0}) // invert the Y axis
+	cam = cam.Scaled(center, s.zoom)                 // apply current zoom factor
+	//cam = cam.Moved(center.Sub(s.camPos))           // Make it central
+	//cam = cam.Rotated(center, -0.785) // Apply isometric angle
+	s.cam = cam
+	pWin.SetMatrix(s.cam)
+
+	pWin.Clear(colornames.Black)
+	pixel.NewSprite(sprite.Pic, sprite.Pic.Bounds()).Draw(pWin, pixel.IM.Moved(center))
+}
+
+func (s *state) handleKeys(pWin *pixelgl.Window) {
+
+	if pWin.JustPressed(pixelgl.KeyMinus) {
+		if s.spriteIdx > 0 {
+			s.spriteIdx -= 1
+		}
+	}
+
+	if pWin.JustPressed(pixelgl.KeyEqual) {
+		if s.spriteIdx < len(s.env.obj.Sprites)-1 {
+			s.spriteIdx += 1
+		}
+	}
+
+	// Zoom in and out with the mouse wheel
+	s.zoom *= math.Pow(1.2, pWin.MouseScroll().Y)
+}

doc/formats/obj.md

@@ -242,30 +242,6 @@ Investigating the sprite data a little more, it seems that we tend to have Y
 null-separated records: 1 record for sprites with a height of 1, 100 for those
 with a height of 100, etc.
 
-The first byte of each record seems likely to specify format - it's mostly
-invariant (always 1 for the 1x1 examples, almost always 0x80 for the altar
-sprites and the majority of other .obj files). Some other values appear too, but
-not a wide distribution.
-
-Number of bytes per row varies. `altar.obj` is diamond-shaped, and the widths
-are also diamond-shaped, so perhaps 0x80 means "center the data around the X
-origin and assume anything unspecified is transparent"?
-
-Result, with my rendering according to those rules on the left and WH40K_TD.exe
-on the right:
-
-![Altar sprite 0 rendering attempt](img/altar_sprite_0.png)
-
-Hurrah!
-
-"Borrowing" the 256-colour palette from `Pic/wh40k.pcx`, I get:
-
-![Altar sprite 0 with borrowed palette](img/altar_sprite_0_paletted.png)
-
-It's still not perfect. Comparing the records in sprite 0 (blank) with those in
-sprite 1 (diamond)....
-
-
 Blank:
 ```
 0x0000 d1 00 42 01
@@ -277,8 +253,8 @@ Blank:
 
 jungtil sprite 0 (blank, draws nothing to screen
 
-0x0018 80 3e 04 1f 80 3e 00 = row 1
+0x0018 80 3e 04 1f 80 3e 00 128 062 004 031 128 062 000
-       80 3c 08 1f 80 3c 00
+       80 3c 08 1f 80 3c 00 128 060 008 031 128 060 000
        80 3a 0c 1f 80 3a 00
        80 38 10 1f 80 38 00
        80 36 14 1f 80 36 00
@@ -307,9 +283,9 @@ jungtil sprite 0 (blank, draws nothing to screen
        80 08 70 1f 80 08 00
        80 06 74 1f 80 06 00
        80 04 78 1f 80 04 00
-       80 02 7c 1f 80 02 00
+       80 02 7c 1f 80 02 00      128 002 124 031 128 002 000  = row 31
-       7f 1f 01 1f 00            = row 32
+       7f 1f 01 1f 00            127 031 001 031 000          = row 32
-       80 02 7c 1f 80 02 00
+       80 02 7c 1f 80 02 00      128 002 124 031 128 002 000  = roe 33
        80 04 78 1f 80 04 00
        80 06 74 1f 80 06 00
        80 08 70 1f 80 08 00
@@ -349,7 +325,8 @@ Sprite 1:
 0x0008 00 00 00 00 5a 11 00 00
 0x0010 00 00 00 00 00 00 00 00
 
-0x0018 80 3e 84 6d 6c 6e 1e 80 3e 00
+0x0018 80 3e 84 6d 6c 6e 1e 80 3e 00  128 062 132 109 108 110 30 128 062 000
+
        80 3c 88 bf 76 6e 6d 6e 76 76 6e 80 3c 00
 0x0030 80 3a 84 bf 76 6e 76 04 6d 84 6e 76 7d 97 80 3a 00
        80 38 86 6d 76 6e 76 6e 87 04 6d 86 6e 97 1e 6e 6e 97 80 38 00
@@ -365,13 +342,18 @@ Sprite 1:
        80 24 81 76 03 97 b4 6e 97 97 6e 97 6e 97 97 1d 97 7f 6e 97 6e 1c 6e 97 6e 1e 76 7f bf 6e 97 1e 76 6e 6e 1e 97 6d 6e 97 1e 97 6e 97 6d 97 be 6e 87 6e 97 1c 97 1d 97 97 6c 97 6d 80 24 00
        80 22 b7 bf 1e 6d 6d 6e 97 97 96 76 5f 1c 87 97 97 1e 97 6e 4f 1e 76 97 1e 97 1e 7e 97 1c 1e 1e 6c 6d 97 6d 76 6e 97 1e 1e 7e 1e 97 6d 76 6e ad 87 1c 6d 87 97 1d 87 97 be 97 03  6d 82 6e 97 80 22 00
        80 20 81 bf 03 97 83 6e 97 6e 03 97 b6 1c 1c 6d ad 6e 97 1e 6e 97 76 6c 8f 6d 6c 96 97 1e 97 1d 97 1e 76 6e 6d 76 6e 6d 97 1e bf 1d 76 6e 97 6e 76  87 6e 87 97 96 97 6e 97 6d 6e 6e 76 6e 87 6c 6d 6d 6e 80 20 00
-0x298 ...
+       80 1e c4 6c 87 76 6e 97 97 6e 1c 97 97 6d 97 97 6e 97 1e 97 1e 1c 6d 6d 87 76 6d be 76 6d 97 1c 97 1e bf 76 7d 97 1d 76 96 6e 6e 97 1e 97 97 6e 97 87 6e 97 97 87 76 87 6e 1c 97  6e ad 1e 6e 97 6c 76 6d 87 6d 1c 76 80 1e 00
+       80 1c c8 bf 97 76 97 76 ad 6e 6d 97 97 6e 6e 76 6e 97 97 1e 1c 97 97 4f 1e 76 6d 6d be 6d 6d be 7d 
+
+
 ```
 
-So the first *and last* two bytes in each record are invariant between the two
+The first byte of each record seems width-related. Mostly matches sprite width,
-tiles, but the interstitial data differs. So we can make a first pass at
+at least in the cases I've looked at so far. Not in the 0x01 case, but certainly
-improving matters by just ignoring those extra bytes for now. Do they say what
+in the 0x80 case, we then get a byte that seems to specify an offset for the
-the Y offset is? Why repeat it?
+following pixeldata, and then a trailing pair of bytes with the same value.
+
+For these tiles, the central (widest) row has 0x7f instead of 0x80.
 
 ## Debugger
 

internal/conv/object.go

@@ -1,7 +1,7 @@
 package conv
 
 import (
-	"bytes"
+	"fmt"
 	"image/color"
 	"log"
 
@@ -34,7 +34,7 @@ func ConvertObject(rawObj *data.Object, name string) *Object {
 	}
 
 	for i, rawSpr := range rawObj.Sprites {
-		pic := spriteToPic(rawSpr)
+		pic := spriteToPic(name, i, rawSpr)
 		out.Sprites[i] = Sprite{
 			Width:  int(rawSpr.Width),
 			Height: int(rawSpr.Height),
@@ -49,53 +49,81 @@ func ConvertObject(rawObj *data.Object, name string) *Object {
 var transparent = color.RGBA{0, 0, 0, 0}
 
 // WIP. Try to convert the pixeldata into a picture.
-func spriteToPic(sprite *data.Sprite) *pixel.PictureData {
+func spriteToPic(name string, idx int, sprite *data.Sprite) *pixel.PictureData {
 	pic := pixel.MakePictureData(pixel.R(float64(0), float64(0), float64(sprite.Width), float64(sprite.Height)))
 
-	buf := bytes.NewBuffer(sprite.PixelData)
+	log.Printf("%v %v: width=%v height=%v", name, idx, sprite.Width, sprite.Height)
 
-	// The pixeldata seems to be formed of Y null-terminated records, with
-	// varying numbers of bytes in each row. Probably [type, *data] but ignore
-	// type for now.
-	//
-	// Theory: perhaps the data in each X is centered around the origin?
 	for y := 0; y < int(sprite.Height); y++ {
-		insn := buf.Next(1)[0] // Take the instruction byte, if that's what it is
+		// Start with all bytes transparent
-
+		for x := 0; x < int(sprite.Width); x++ {
-		switch insn {
+			pic.Pix[pic.Index(pixel.V(float64(x), float64(y)))] = transparent
-		case 0:
-			log.Printf("Reached the end of the sprite at y=%v (height=%v)", y, sprite.Height)
-		case 1, 0x80:
-			// Ignore these, as we know they exist and the logic below seems to handle them
-			// Although I suspect 0x80 means "centered run of bytes" while 0x1
-			// means "left-aligned run of bytes", since all 0x1 rows seem to be
-			// for 1x1 images, it makes no practical difference
-		default:
-			log.Printf("Record of unknown type %v", insn)
 		}
 
-		rowData, err := buf.ReadBytes(0)
+		row := sprite.Rows[y]
-		if err != nil {
+		log.Printf("%#v", row)
-			log.Printf("Error at y=%d: %v", y, err)
+		pixels := row[0 : len(row)-1] // Strip off the record separator (0x00)
-			continue
+
+		// Not really clear on what this does yet. Aligned with sprite width in
+		// many cases but can also vary above and below that value.
+		u0 := int(pixels[0])
+		pixels = pixels[1:len(pixels)]
+
+		// In some cases, the column data is indented relative to the start of
+		// the row. Certainly true when u0 == 0x80, perhaps in other cases
+		// too.
+		//
+		// Definitely not the case when u0 == 0x01 - there aren't enough bytes
+		// in that case for it to be anything but pixeldata
+		xOffset := 0
+
+		// Do nothing if we're out of pixels
+		if u0 == 0x80 {
+			log.Printf("Handling 0x80: %#v", pixels)
+			xOffset = int(pixels[0])
+			pixels = pixels[1:len(pixels)]
+
+			// Sometimes, pixels is now empty. e.g. l_ivy02 sprite 6
+			if len(pixels) > 3 {
+
+				// For tiles, this has an inverse relationship with u0. Seems to add
+				// up to 0x42 in all cases, which matches byte 3 of the header?
+				//_ = int(pixels[0])
+
+				pixels = pixels[1:len(pixels)]
+
+				// On tiles, this removes some junk around the edge, but doesn't
+				// seem to be reasonable in-general?
+				pixels = pixels[0 : len(pixels)-2]
+			}
 		}
 
-		// Ignore the record separator
+		log.Printf(
-		rowData = rowData[0 : len(rowData)-1]
+			"%v %d: len(row)=%v, len(pixels)=%v sprWidth=%v u0=%v xOffset=%v",
-		leftPad := (int(sprite.Width) - len(rowData)) / 2
+			name, idx, len(row), len(pixels), sprite.Width, u0, xOffset,
+		)
 
-		// Set all bytes to be transparent by default
+		for x, b := range pixels {
-		for allX := 0; allX < int(sprite.Width); allX++ {
+			vec := pixel.V(float64(xOffset+x), float64(y))
-			idx := pic.Index(pixel.V(float64(allX), float64(y)))
+			if err := setPaletteColor(pic, vec, b); err != nil {
-			pic.Pix[idx] = transparent
+				log.Printf("%s %d: %d,%d: %v", name, idx, x, y, err)
-		}
+			}
-
-		for x, b := range rowData {
-			idx := pic.Index(pixel.V(float64(leftPad+x), float64(y)))
-			r, g, b, a := data.ColorPalette[int(b)].RGBA()
-			pic.Pix[idx] = color.RGBA{uint8(r), uint8(g), uint8(b), uint8(a)}
 		}
 	}
 
 	return pic
 }
+
+func setPaletteColor(pic *pixel.PictureData, point pixel.Vec, colorIdx byte) error {
+	idx := pic.Index(point)
+
+	if idx > len(pic.Pix)-1 {
+		return fmt.Errorf("Got index %v which exceeds bounds", idx)
+	}
+
+	r, g, b, a := data.ColorPalette[int(colorIdx)].RGBA()
+	color := color.RGBA{uint8(r), uint8(g), uint8(b), uint8(a)}
+
+	pic.Pix[idx] = color
+	return nil
+}

internal/data/object.go

@@ -1,7 +1,7 @@
 package data
 
 import (
-	"bytes"
+	"bufio"
 	"encoding/binary"
 	"fmt"
 	"io"
@@ -36,7 +36,7 @@ func (s SpriteHeader) Check(expectedSize uint32) error {
 type Sprite struct {
 	SpriteHeader
 
-	PixelData []byte
+	Rows [][]byte
 }
 
 type dirEntry struct {
@@ -113,27 +113,18 @@ func LoadObject(filename string) (*Object, error) {
 		return nil, err
 	}
 
-	// FIXME: this might - *might* - load interstitial data we don't really
-	// need, so wasting memory.
-	data, err := ioutil.ReadAll(f)
-	if err != nil {
-		return nil, err
-	}
-
-	buf := bytes.NewReader(data)
-
 	for _, dirEntry := range dir {
 		if err := dirEntry.Check(); err != nil {
 			return nil, err
 		}
 
-		if _, err := buf.Seek(int64(dirEntry.Offset), io.SeekStart); err != nil {
+		if _, err := f.Seek(int64(out.DataOffset+dirEntry.Offset), io.SeekStart); err != nil {
 			return nil, err
 		}
 
 		sprite := &Sprite{}
 
-		if err := binary.Read(buf, binary.LittleEndian, &sprite.SpriteHeader); err != nil {
+		if err := binary.Read(f, binary.LittleEndian, &sprite.SpriteHeader); err != nil {
 			return nil, err
 		}
 
@@ -141,11 +132,18 @@ func LoadObject(filename string) (*Object, error) {
 			return nil, err
 		}
 
-		// It's safe to assume that a `bytes.Reader` will always satisfy the
+		// The pixeldata seems to be formed of Y null-terminated records, with
-		// requested read size.
+		// varying numbers of bytes in each row. I don't know the internal
-		sprite.PixelData = make([]byte, sprite.PixelSize)
+		// structure yet, but there's definitely working pixel data in there
-		if _, err := buf.Read(sprite.PixelData); err != nil {
+		buf := bufio.NewReader(io.LimitReader(f, int64(sprite.PixelSize)))
-			return nil, err
+		sprite.Rows = make([][]byte, sprite.Height)
+
+		for y := 0; y < int(sprite.Height); y++ {
+			if row, err := buf.ReadBytes(0x00); err != nil {
+				return nil, err
+			} else {
+				sprite.Rows[y] = row
+			}
 		}
 
 		out.Sprites = append(out.Sprites, sprite)

scripts/try-uncompress

@@ -138,7 +138,7 @@ module Obj
       sprite_pixels = rel_data[hdr.pixel_range]
 
       records = sprite_pixels.split("\x00")
-      records.map { |record| record += "\x00" }
+      records.map! { |record| record += "\x00" }
 
       new(hdr, records, rel_data)
     end
@@ -216,8 +216,8 @@ def display(data, blocksize=8, skip=0, header: false)
     out = [
       "0x#{hex(i*blocksize, 4)}",
       block.map { |b| hex(b, 2) },                          # hex
-      "    | " + block.map { |b| text(b) }.join("") + " |",   # ascii
+ #     "    | " + block.map { |b| text(b) }.join("") + " |",   # ascii
-      block.map { |b| ascii(b) } ,# decimal bytes
+#      block.map { |b| ascii(b) } ,# decimal bytes
       "",# decimal 2-bytes
      # decimal 4-bytes
     ]
@@ -273,7 +273,6 @@ def decompress(filename)
       break if data.empty?
 
       right = data.index(0)
-      
 
       if right.nil?
         print "error! end of chunk not found"
@@ -360,9 +359,35 @@ def sprites(filename)
   end
 end
 
+def sprite(filename, i)
+  obj = load_obj(filename)
+
+  puts filename + ":"
+  spr = obj.sprites[i]
+  raise "No sprite #{i}" unless spr
+
+  # Show the header
+  display(spr.raw[0...24], 4, header: true)
+
+  spr.records.each_with_index do |record, i|
+    str =
+      if record.size > 40
+        record[0...20].bytes.map { |b| hex(b, 2) }.join(" ") +
+        " ... " +
+        record[-20..-1].bytes.map { |b| hex(b, 2) }.join(" ")
+      else
+        record.bytes.map { |b| hex(b, 2) }.join(" ")
+      end
+
+    puts "%4d: %4d bytes: %s"%[i,record.size, str]
+  end
+end
+
 case command = ARGV.shift
 when "sprites" then
   ARGV.each { |filename| sprites(filename) } 
+when "sprite" then
+  sprite(ARGV[0], ARGV[1].to_i)
 when "dump" then
   ARGV.each { |filename| dump(filename) } 
 when "compare" then