Determine the RLE format for .obj file sprite pixeldata

.gitignore

@@ -1,5 +1,6 @@
 /loader
 /orig
+/palette-idx
 /view-obj
 /view-map
 /view-minimap

Makefile

@@ -1,10 +1,13 @@
 srcfiles = $(shell find . -iname *.go)
 
-all: loader view-obj view-map view-minimap view-set
+all: loader palette-idx view-obj view-map view-minimap view-set
 
 loader: $(srcfiles)
 	go build -o loader ur.gs/ordoor/cmd/loader
 
+palette-idx: $(srcfiles)
+	go build -o palette-idx ur.gs/ordoor/cmd/palette-idx
+
 view-obj: $(srcfiles)
 	go build -o view-obj ur.gs/ordoor/cmd/view-obj
 

README.md

@@ -42,9 +42,10 @@ The `view-map` binary attempts to render a map, and is the current focus of
 effort. Once I can render a whole map, including pre-placed characters (cultist
 scum), things can start to get more interesting.
 
-Current status: map tiles are rendered at correct offsets. Tiles (four per 
-coordinate: floor, centre, left, and right) are rendered, but the pixels for
-those tiles are not 100% accurate.
+Current status: map tiles are rendered at correct offsets. Static objects (four
+per map  coordinate: floor, centre, left, and right) are rendered mostly fine,
+although objects at each Z level don't *quite* stack correctly on top of each
+other yet.
 
 Characters and animations aren't touched at all yet. Rendering performance is
 atrocious. No gameplay, no sound, no campaign logic. Interaction with the play
@@ -59,6 +60,10 @@ $ make view-map
 $ ./view-map -map orig/Maps/Chapter01.MAP -txt orig/Maps/Chapter01.TXT
 ```
 
+Looks like this:
+
+![](doc/formats/img/chapter01_rendered_2018-09-08.png)
+
 Use the arrow keys to scroll around the map, the mouse wheel to zoom, and the
 `1` - `7` keys to change Z level.
 

cmd/palette-idx/main.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+
+	"ur.gs/ordoor/internal/data"
+)
+
+func main() {
+	switch os.Args[1] {
+	case "index", "i":
+		idx, err := strconv.ParseInt(os.Args[2], 16, 64)
+		if err != nil {
+			fmt.Println("Usage: palette-idx i <0-255>")
+			os.Exit(1)
+		}
+
+		fmt.Printf("Palette[%v]: %#v\n", idx, data.ColorPalette[idx])
+	case "color", "colour", "c":
+		fmt.Println("TODO!")
+		os.Exit(1)
+	}
+
+}

doc/formats/obj.md

@@ -85,8 +85,8 @@ The `type` field **may** tell us what format each sprite is in.
 `.obj` files represent visual data. They contain a number of sprites, which are
 assigned attributes in `.asn` files and referenced from `.map` files.
 
-The container format is worked out, but the per-sprite data is still unknown, so
-I'm documenting the former here while still investigating the latter.
+The container format is worked out, but the per-sprite data is still WIP, so I'm
+documenting the former here while still investigating the latter.
 
 The file begins with a header, with all values 32-bit little-endians:
 
@@ -357,6 +357,149 @@ following pixeldata, and then a trailing pair of bytes with the same value.
 
 For these tiles, the central (widest) row has 0x7f instead of 0x80.
 
+
+## Comparative rendering
+
+Here, I'm focusing on the `altar.obj` file. It has just two sprites and is a
+`CENTER` object. Here's how it looks when rendered by `WH40K_TD.exe` (there's
+a cloud of uncertainty around the skulls at present):
+
+![](img/altar.obj.original.png)
+
+And here's the rendering of the first sprite, done by my code as of 2018-09-07:
+
+![](img/altar.obj.me-2018-09-07.png)
+
+Initial impressions:
+
+* The original clearly has some transparent pixels
+* Skulls are scrambled, more so to the right
+* Starting X-axis along the top half is well out
+* The tabletop has become basically invisible
+
+I think some sort of RLE *must* be in place for repeated pixels in a line.
+
+Digging into the pixeldata, we have 3 distinct stages in the first 16 records
+(which are null-terminated):
+
+1. Top of skull
+1. Right hand side is marked by a vertical antenna-like thing
+1. The stepped altar resumes on the right.
+
+Header:
+
+```
+------------------
+        0  1  2  3
+------------------
+0x0000 d6 00 16 01 
+0x0004 74 00 67 00 (x = 116, y = 103)
+0x0008 00 00 00 00 
+0x000c 11 1d 00 00 
+0x0010 00 00 00 00 
+0x0014 00 00 00 00 
+```
+
+
+Here's the first row of pixeldata:
+
+```
+80 3d 82 65 1d 80 35 00 (0)
+```
+
+There are just two visible pixels in this row, and they are specified correctly
+by the middle two bytes.
+
+The 3 on either side are framing data, for certain. It's not clear exactly what
+they specify though.
+
+Onto the next row:
+
+```
+80 38 87 97 19 17 18 17 18 16 80 35 00 (1)
+```
+
+These pixels end in the same position on the right, but start 5 pixels earlier.
+So there are 7 pixels total, specified by the middle bytes again.
+
+The RHS bytes are identical. The LHS bytes are 5 lower and higher, respectively.
+
+Hmm. More rows, not analyzed in depth yet:
+
+```
+80 37 82 17 19 03 17 04 16                        80 34 00 ( 2)
+80 36 84 18 16 15 15 04 16 83 14 14 16            80 33 00 ( 3)
+80 35 83 17 17 16 03 17 81 18 03 17 83 a8 16 17   80 32 00 ( 4)
+
+80 35 8e            18 19 aa 18 17 17 18 18 17 a8 a8 17 18 18   80 02   81 ad   80 2e   00 ( 5) - right vertical bar starts
+80 34 8f         18 ab 19 18 aa 19 19 ab ab 19 aa aa 17 17 19   80 02   81 ad   80 2e   00 ( 6)
+80 34 8c      19 ab 19 19 1a ad ad 1b ad ad 1a ab 03 17 81 19   80 01   81 ad   80 2e   00 ( 7)
+80 34 90      ac 19 18 18 ab 1b ae 1c ae 1c ad ab 17 16 aa 1b   80 01   81 ae   80 2e   00 ( 8)
+80 33 91   1a 1c ac 18 19 1a 1b 1c ae 1c 1c ad 1a 19 17 19 ae   80 01   81 ae   80 2e   00 ( 9)
+80 33 91   1a 1c ad 1a ad 1c ae 1c ae 1c 1c ad 1c ad 1a ab ad   80 01   81 ae   80 2e   00 (10)
+
+80 31 81   8b   80 01 91        19 1c 1a ad ae 1d 1c af 1d 1c af 1c ae ae 1b ab ad       80 01 81   af   80 2e 00 (11) - left vertical bar starts
+80 31 81   ab   80 02 86        1c ad 1b 1c 1c af 04 1d 86 1c ae 1b ad 1a ae             80 01 81   af   80 2e 00 (12)
+80 31    81 ad    80 01    93 1f 1c 1c 1b 1c 1b ae 1d 1c 1d ae 1c ad ad 1b ae 1c 1f ad                 80 2e 00 (13)
+```
+
+Now for something more interesting. At line 14, an antenna starts to sprout on
+the far left. It is one pixel wide, followed by 19 transparent pixels, followed
+by another antenna and the skull. 
+
+```
+  1        2        3        4 
+80 1d    81 ad    80 13    85 ac 1f 1f af 1d     08 1c          89 1b 1b 1c ae 1c 1a 1f af 1f                       80 2d 00 (14) - right vertical bar is no longer the last thing
+80 1d    81 ac    80 11    83 1f 1f ac           03 1f          81 af                            09 1d 87 1c 1d ae 1d 1f 1f ae 03 1f                 80 2b 00 (15)
+80 1d    81 ac    80 0f    04 1f                 81 ab 03 1f    88 1c 1d 17 15 15 1d 1e 1c       05 1d 03 1f 81 af 05 1f 80 29 00
+
+```
+
+Looking at `y=14`, the first couplet *must* define where we start, and
+that all bytes before then are transparent. Looks like `0x80 <count>`
+
+The second couplet writes 1 byte in a defined colour. Palette indices 0xad and
+0xac match the pixels rendered by `WH40K_TD.exe`.
+
+The third couplet defines the transparent section between the far left antenna
+and the left antenna or table. Like the first couplet, it is `0x80 <count>`.
+
+At 4, the couplet idea starts to break down, but this is definitely starting to
+look RLE-ish.
+
+Let's go back to line 2 with the knowledge above. We have 9 literal bytes to
+display. Decoded, they should equal:
+
+```
+17 19 17 17 17 16 16 16 16
+```
+
+(these are the indexes for the colours each pixel is, as rendered by
+`WH40K_TD.EXE`).
+
+The encoded bytes:
+
+```
+80 37    82 17 19    03 17    04 16     80 34    00 ( 2)
+```
+
+We can get to the output by applying these rules to each byte of encoded data:
+
+* `< 0x80`: repeat the next byte that many times
+* `= 0x80`: skip `<count>` bytes
+* `> 0x80`: take literal - 0x80 bytes
+
+Doing so, the altar displays, but more importantly, so does all of
+`Chapter01.MAP`!
+
+![](img/chapter01_rendered_2018-09-08.png)
+
+The annoying "static" is gone, and the only rendering issue visible is that the
+objects from different Z levels don't stack correctly. Hooray for progress!
+
+I haven't tried any of the other objects in different chapters yet. Perhaps some
+of them will fail to render?
+
 ## Debugger
 
 

internal/conv/object.go

@@ -52,60 +52,52 @@ var transparent = color.RGBA{0, 0, 0, 0}
 func spriteToPic(name string, idx int, sprite *data.Sprite) *pixel.PictureData {
 	pic := pixel.MakePictureData(pixel.R(float64(0), float64(0), float64(sprite.Width), float64(sprite.Height)))
 
-	//log.Printf("%v %v: width=%v height=%v", name, idx, sprite.Width, sprite.Height)
+	log.Printf("%v %v: width=%v height=%v", name, idx, sprite.Width, sprite.Height)
 
 	for y := 0; y < int(sprite.Height); y++ {
+		encoded := sprite.Rows[y]
+		decoded := make([]byte, 0, int(sprite.Width))
+
 		// Start with all bytes transparent
 		for x := 0; x < int(sprite.Width); x++ {
 			pic.Pix[pic.Index(pixel.V(float64(x), float64(y)))] = transparent
 		}
 
-		row := sprite.Rows[y]
-		//log.Printf("%#v", row)
-		pixels := row[0 : len(row)-1] // Strip off the record separator (0x00)
+		for i := 0; i < len(encoded); i++ {
+			b := encoded[i]
 
-		// Not really clear on what this does yet. Aligned with sprite width in
-		// many cases but can also vary above and below that value.
-		u0 := int(pixels[0])
-		pixels = pixels[1:len(pixels)]
+			// This appears to be a kind of RLE
+			if b == 0 {
+				continue // finished
+			} else if b < 0x80 {
+				// repeat the next byte this many times
+				for j := 0; j < int(b); j++ {
+					decoded = append(decoded, encoded[i+1])
+				}
 
-		// In some cases, the column data is indented relative to the start of
-		// the row. Certainly true when u0 == 0x80, perhaps in other cases
-		// too.
-		//
-		// Definitely not the case when u0 == 0x01 - there aren't enough bytes
-		// in that case for it to be anything but pixeldata
-		xOffset := 0
+				i++ // skip the repeat byte
+			} else if b == 0x80 {
+				// transparent value, skip forward *x+1 rows
+				skip := int(encoded[i+1])
+				for i := 0; i < skip; i++ {
+					decoded = append(decoded, byte(0x00))
+				}
 
-		// Do nothing if we're out of pixels
-		if u0 == 0x80 {
-			//log.Printf("Handling 0x80: %#v", pixels)
-			xOffset = int(pixels[0])
-			pixels = pixels[1:len(pixels)]
+				i++ // skip the count byte
+			} else {
+				// take the next b-0x80 bytes literally
+				literals := int(b) - 0x80
+				for j := i + 1; j <= i+literals; j++ {
+					decoded = append(decoded, encoded[j])
+				}
 
-			// Sometimes, pixels is now empty. e.g. l_ivy02 sprite 6
-			if len(pixels) > 3 {
-
-				// For tiles, this has an inverse relationship with u0. Seems to add
-				// up to 0x42 in all cases, which matches byte 3 of the header?
-				//_ = int(pixels[0])
-
-				pixels = pixels[1:len(pixels)]
-
-				// On tiles, this removes some junk around the edge, but doesn't
-				// seem to be reasonable in-general?
-				pixels = pixels[0 : len(pixels)-2]
+				i = i + literals
 			}
 		}
 
-		//log.Printf(
-		//	"%v %d: len(row)=%v, len(pixels)=%v sprWidth=%v u0=%v xOffset=%v",
-		//	name, idx, len(row), len(pixels), sprite.Width, u0, xOffset,
-		//)
-
-		for x, b := range pixels {
-			vec := pixel.V(float64(xOffset+x), float64(y))
-			if err := setPaletteColor(pic, vec, b); err != nil {
+		// Update the picture
+		for x, b := range decoded {
+			if err := setPaletteColor(pic, x, y, b); err != nil {
 				log.Printf("%s %d: %d,%d: %v", name, idx, x, y, err)
 			}
 		}
@@ -114,8 +106,9 @@ func spriteToPic(name string, idx int, sprite *data.Sprite) *pixel.PictureData {
 	return pic
 }
 
-func setPaletteColor(pic *pixel.PictureData, point pixel.Vec, colorIdx byte) error {
-	idx := pic.Index(point)
+func setPaletteColor(pic *pixel.PictureData, x int, y int, colorIdx byte) error {
+	vec := pixel.V(float64(x), float64(y))
+	idx := pic.Index(vec)
 
 	if idx > len(pic.Pix)-1 {
 		return fmt.Errorf("Got index %v which exceeds bounds", idx)