Lazily load sprite image data

This cuts memory use significantly, since many sprites in an object are
never used. We can get savings over time by evicting sprites when they
go out of scope, but that's, well, out of scope.

To achieve this, I introduce an assetstore package that is in charge of
loading things from the filesystem. This also allows some lingering
case-sensitivity issues to be handled cleanly.

I'd hoped that creating fewer ebiten.Image instances would help CPU
usage, but that doesn't seem to be the case.

cmd/view-map/main.go

@@ -2,32 +2,25 @@ package main
 
 import (
 	"flag"
-	"fmt"
 	"image"
 	"log"
 	"math"
 	"os"
-	"path/filepath"
 
 	"github.com/hajimehoshi/ebiten"
 
-	"code.ur.gs/lupine/ordoor/internal/conv"
+	"code.ur.gs/lupine/ordoor/internal/assetstore"
-	"code.ur.gs/lupine/ordoor/internal/data"
-	"code.ur.gs/lupine/ordoor/internal/maps"
-	"code.ur.gs/lupine/ordoor/internal/sets"
 	"code.ur.gs/lupine/ordoor/internal/ui"
 )
 
 var (
 	gamePath = flag.String("game-path", "./orig", "Path to a WH40K: Chaos Gate installation")
-	mapFile  = flag.String("map", "", "Prefix path to a .map file, e.g. ./orig/Maps/Chapter01.MAP")
+	gameMap  = flag.String("map", "", "Name of a map, e.g., Chapter01")
-	txtFile  = flag.String("txt", "", "Prefix path to a .txt file, e.g. ./orig/Maps/Chapter01.txt")
 )
 
 type env struct {
-	gameMap *maps.GameMap
+	assets *assetstore.AssetStore
-	set     *sets.MapSet
+	area   *assetstore.Map
-	objects map[string]*conv.Object
 
 	step      int
 	state     state
@@ -43,39 +36,19 @@ type state struct {
 func main() {
 	flag.Parse()
 
-	if *gamePath == "" || *mapFile == "" || *txtFile == "" {
+	if *gamePath == "" || *gameMap == "" {
 		flag.Usage()
 		os.Exit(1)
 	}
 
-	gameMap, err := maps.LoadGameMapByFiles(*mapFile, *txtFile)
+	assets, err := assetstore.New(*gamePath)
 	if err != nil {
-		log.Fatalf("Couldn't load map file: %v", err)
+		log.Fatalf("Failed to scan root directory %v: %v", *gamePath, err)
 	}
 
-	setFile := filepath.Join(*gamePath, "Sets", gameMap.MapSetFilename())
+	area, err := assets.Map(*gameMap)
-	log.Println(setFile)
-	mapSet, err := sets.LoadSet(setFile)
 	if err != nil {
-		log.Fatalf("Couldn't load set file %s: %v", setFile, err)
+		log.Fatalf("Failed to load map %v: %v", *gameMap, err)
-	}
-
-	objects := make([]*conv.Object, 0, len(mapSet.Palette))
-	for _, name := range mapSet.Palette {
-		objFile := filepath.Join(*gamePath, "Obj", name+".obj")
-		rawObj, err := data.LoadObject(objFile)
-		if err != nil {
-			log.Fatalf("Failed to load %s: %v", name, err)
-		}
-
-		rawObj.Name = name
-
-		obj, err := conv.ConvertObject(rawObj, name)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		objects = append(objects, obj)
 	}
 
 	state := state{
@@ -83,14 +56,13 @@ func main() {
 		origin: image.Point{0, 3000}, // FIXME: haxxx
 	}
 	env := &env{
-		gameMap:   gameMap,
+		area:      area,
-		set:       mapSet,
+		assets:    assets,
-		objects:   conv.MapByName(objects),
 		state:     state,
 		lastState: state,
 	}
 
-	win, err := ui.NewWindow("View Map " + *mapFile)
+	win, err := ui.NewWindow("View Map " + *gameMap)
 	if err != nil {
 		log.Fatal("Couldn't create window: %v", err)
 	}
@@ -123,30 +95,6 @@ func (e *env) Update() error {
 	return nil
 }
 
-func (e *env) getSprite(palette []string, ref maps.ObjRef) (*conv.Sprite, error) {
-	// There seems to be an active bit that hides many sins
-	if !ref.IsActive() {
-		return nil, nil
-	}
-
-	if ref.Index() >= len(palette) {
-		return nil, fmt.Errorf("Palette too small: %v requested", ref.Index())
-	}
-
-	name := palette[ref.Index()]
-
-	obj := e.objects[name]
-	if obj == nil {
-		return nil, fmt.Errorf("Failed to find surface sprite %#v -> %q", ref, name)
-	}
-
-	if ref.Sprite() >= len(obj.Sprites) {
-		return nil, fmt.Errorf("Out-of-index sprite %v requested for %v", ref.Sprite(), name)
-	}
-
-	return obj.Sprites[ref.Sprite()], nil
-}
-
 func (e *env) Draw(screen *ebiten.Image) error {
 	// Bounds clipping
 	// http://www.java-gaming.org/index.php?topic=24922.0
@@ -177,8 +125,7 @@ func (e *env) Draw(screen *ebiten.Image) error {
 			x := (a + b) / 2
 			y := (a - b) / 2
 
-			if x < int(e.gameMap.MinWidth) || x >= int(e.gameMap.MaxWidth) ||
+			if !image.Pt(x, y).In(e.area.Rect) {
-				y < int(e.gameMap.MinLength) || y >= int(e.gameMap.MaxLength) {
 				continue
 			}
 
@@ -192,31 +139,9 @@ func (e *env) Draw(screen *ebiten.Image) error {
 }
 
 func (e *env) renderCell(x, y, z int, screen *ebiten.Image) error {
-	var sprites []*conv.Sprite
+	images, err := e.area.ImagesForCell(x, y, z)
-	cell := e.gameMap.Cells.At(x, y, z)
+	if err != nil {
-
+		return err
-	if spr, err := e.getSprite(e.set.Palette, cell.Surface); err != nil {
-		log.Printf("%v %v %v surface: %v", x, y, z, err)
-	} else if spr != nil {
-		sprites = append(sprites, spr)
-	}
-
-	if spr, err := e.getSprite(e.set.Palette, cell.Center); err != nil {
-		log.Printf("%v %v %v center: %v", x, y, z, err)
-	} else if spr != nil {
-		sprites = append(sprites, spr)
-	}
-
-	if spr, err := e.getSprite(e.set.Palette, cell.Left); err != nil {
-		log.Printf("%v %v %v left: %v", x, y, z, err)
-	} else if spr != nil {
-		sprites = append(sprites, spr)
-	}
-
-	if spr, err := e.getSprite(e.set.Palette, cell.Right); err != nil {
-		log.Printf("%v %v %v right: %v", x, y, z, err)
-	} else if spr != nil {
-		sprites = append(sprites, spr)
 	}
 
 	iso := ebiten.GeoM{}
@@ -231,8 +156,8 @@ func (e *env) renderCell(x, y, z int, screen *ebiten.Image) error {
 
 	// TODO: iso.Scale(e.state.zoom, e.state.zoom) // apply current zoom factor
 
-	for _, sprite := range sprites {
+	for _, img := range images {
-		if err := screen.DrawImage(sprite.Image, &ebiten.DrawImageOptions{GeoM: iso}); err != nil {
+		if err := screen.DrawImage(img, &ebiten.DrawImageOptions{GeoM: iso}); err != nil {
 			return err
 		}
 	}

internal/assetstore/assetstore.go

@@ -0,0 +1,125 @@
+package assetstore
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	RootDir = "" // Used in the entryMap for entries pertaining to the root dir
+)
+
+type entryMap map[string]map[string]string
+
+// type AssetStore is responsible for lazily loading game data when it is
+// required. Applications shouldn't need to do anything except set one of these
+// up, pointing at the game dir root, to access all assets.
+//
+// Assets should be loaded on-demand to keep memory costs as low as possible.
+// Cross-platform differences such as filename case sensitivity are also dealt
+// with here.
+//
+// We assume the directory is read-only. You can run Refresh() if you make a
+// change.
+type AssetStore struct {
+	RootDir string
+
+	// Case-insensitive file lookup.
+	// {"":{"anim":"Anim", "obj":"Obj", ...}, "anim":{ "warhammer.ani":"WarHammer.ani" }, ...}
+	entries entryMap
+
+	// These members are used to store things we've already loaded
+	maps map[string]*Map
+	objs map[string]*Object
+	sets map[string]*Set
+}
+
+// New returns a new AssetStore
+func New(dir string) (*AssetStore, error) {
+	store := &AssetStore{
+		RootDir: dir,
+	}
+
+	// fill entryMap
+	if err := store.Refresh(); err != nil {
+		return nil, err
+	}
+
+	return store, nil
+}
+
+func (a *AssetStore) Refresh() error {
+	rootEntries, err := processDir(a.RootDir)
+	if err != nil {
+		return fmt.Errorf("failed to process %v: %v", a.RootDir, err)
+	}
+
+	newEntryMap := make(entryMap, len(rootEntries))
+	newEntryMap[RootDir] = rootEntries
+
+	for lower, natural := range rootEntries {
+		path := filepath.Join(a.RootDir, natural)
+		fi, err := os.Stat(path)
+		if err != nil {
+			return fmt.Errorf("Failed to stat %v: %v", path, err)
+		}
+
+		if fi.IsDir() {
+			entries, err := processDir(path)
+			if err != nil {
+				return fmt.Errorf("Failed to process %v: %v", path, err)
+			}
+
+			newEntryMap[lower] = entries
+		}
+	}
+
+	// Refresh
+	a.entries = newEntryMap
+	a.maps = make(map[string]*Map)
+	a.objs = make(map[string]*Object)
+	a.sets = make(map[string]*Set)
+
+	return nil
+}
+
+func (a *AssetStore) lookup(name, ext string, dirs ...string) (string, error) {
+	filename := canonical(name + "." + ext)
+
+	for _, dir := range dirs {
+		dir = canonical(dir)
+		if base, ok := a.entries[dir]; ok {
+			if file, ok := base[filename]; ok {
+				actualDir := a.entries[RootDir][dir]
+				return filepath.Join(a.RootDir, actualDir, file), nil
+			}
+		}
+	}
+
+	return "", os.ErrNotExist
+}
+
+func canonical(s string) string {
+	return strings.ToLower(s)
+}
+
+func processDir(dir string) (map[string]string, error) {
+	entries, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	out := make(map[string]string, len(entries))
+	for _, entry := range entries {
+		if entry.Name() == "." || entry.Name() == ".." {
+			continue
+		}
+
+		out[canonical(entry.Name())] = entry.Name()
+	}
+
+	return out, nil
+}

internal/assetstore/map.go

@@ -0,0 +1,89 @@
+package assetstore
+
+import (
+	"github.com/hajimehoshi/ebiten"
+	"image"
+
+	"code.ur.gs/lupine/ordoor/internal/maps"
+)
+
+type Map struct {
+	assets *AssetStore
+	set    *Set
+	Rect   image.Rectangle
+
+	raw *maps.GameMap
+}
+
+// Map loads a game map with the given name (e.g. "Chapter01")
+func (a *AssetStore) Map(name string) (*Map, error) {
+	name = canonical(name)
+
+	if m, ok := a.maps[name]; ok {
+		return m, nil
+	}
+
+	mapFile, err := a.lookup(name, "map", "Maps", "MultiMaps")
+	if err != nil {
+		return nil, err
+	}
+
+	txtFile, err := a.lookup(name, "txt", "Maps", "MultiMaps")
+	if err != nil {
+		return nil, err
+	}
+
+	raw, err := maps.LoadGameMapByFiles(mapFile, txtFile)
+	if err != nil {
+		return nil, err
+	}
+
+	// The set for a map is small and frequently referenced, so load it here
+	set, err := a.Set(raw.MapSetName())
+	if err != nil {
+		return nil, err
+	}
+
+	m := &Map{
+		Rect: image.Rect(
+			int(raw.MinWidth),
+			int(raw.MinLength),
+			int(raw.MaxWidth),
+			int(raw.MaxLength),
+		),
+		assets: a,
+		raw:    raw,
+		set:    set,
+	}
+
+	a.maps[canonical(name)] = m
+
+	return m, nil
+}
+
+// ImagesForCell returns the sprites needed to correctly render this cell.
+// They should be rendered from first to last to get the correct ordering
+func (m *Map) ImagesForCell(x, y, z int) ([]*ebiten.Image, error) {
+	cell := m.raw.At(x, y, z)
+	images := make([]*ebiten.Image, 0, 4)
+
+	for _, ref := range []maps.ObjRef{cell.Surface, cell.Center, cell.Left, cell.Right} {
+		if !ref.IsActive() {
+			continue
+		}
+
+		obj, err := m.set.Object(ref.Index())
+		if err != nil {
+			return nil, err
+		}
+
+		img, err := obj.Image(ref.Sprite())
+		if err != nil {
+			return nil, err
+		}
+
+		images = append(images, img)
+	}
+
+	return images, nil
+}

internal/assetstore/object.go

@@ -0,0 +1,64 @@
+package assetstore
+
+import (
+	"github.com/hajimehoshi/ebiten"
+
+	"code.ur.gs/lupine/ordoor/internal/data"
+)
+
+type Object struct {
+	assets *AssetStore
+	images []*ebiten.Image
+
+	raw *data.Object
+}
+
+func (a *AssetStore) Object(name string) (*Object, error) {
+	name = canonical(name)
+
+	if obj, ok := a.objs[name]; ok {
+		return obj, nil
+	}
+
+	filename, err := a.lookup(name, "obj", "Obj")
+	if err != nil {
+		return nil, err
+	}
+
+	raw, err := data.LoadObjectLazily(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	obj := &Object{
+		assets: a,
+		images: make([]*ebiten.Image, int(raw.NumSprites)),
+		raw:    raw,
+	}
+	a.objs[name] = obj
+
+	return obj, nil
+}
+
+// Filled lazily
+func (o *Object) Image(idx int) (*ebiten.Image, error) {
+	if img := o.images[idx]; img != nil {
+		return img, nil
+	}
+
+	if o.raw.Sprites[idx] == nil {
+		if err := o.raw.LoadSprite(idx); err != nil {
+			return nil, err
+		}
+	}
+
+	stdImg := o.raw.Sprites[idx].ToImage()
+	img, err := ebiten.NewImageFromImage(stdImg, ebiten.FilterDefault)
+	if err != nil {
+		return nil, err
+	}
+
+	o.images[idx] = img
+
+	return img, nil
+}

internal/assetstore/set.go

@@ -0,0 +1,50 @@
+package assetstore
+
+import (
+	"errors"
+
+	"code.ur.gs/lupine/ordoor/internal/sets"
+)
+
+var (
+	ErrOutOfBounds = errors.New("Out of bounds")
+)
+
+type Set struct {
+	assets *AssetStore
+
+	raw *sets.MapSet
+}
+
+func (s *Set) Object(idx int) (*Object, error) {
+	if idx < 0 || idx >= len(s.raw.Palette) {
+		return nil, ErrOutOfBounds
+	}
+
+	return s.assets.Object(s.raw.Palette[idx])
+}
+
+func (a *AssetStore) Set(name string) (*Set, error) {
+	name = canonical(name)
+	if set, ok := a.sets[name]; ok {
+		return set, nil
+	}
+
+	filename, err := a.lookup(name, "set", "Sets")
+	if err != nil {
+		return nil, err
+	}
+
+	raw, err := sets.LoadSet(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	set := &Set{
+		assets: a,
+		raw:    raw,
+	}
+	a.sets[name] = set
+
+	return set, nil
+}

internal/conv/object.go

@@ -1,8 +1,6 @@
 package conv
 
 import (
-	"image"
-
 	"github.com/hajimehoshi/ebiten"
 
 	"code.ur.gs/lupine/ordoor/internal/data"
@@ -43,8 +41,7 @@ func ConvertObject(rawObj *data.Object, name string) (*Object, error) {
 	for i, rawSpr := range rawObj.Sprites {
 		w := int(rawSpr.Width)
 		h := int(rawSpr.Height)
-		stdImage := spriteToImage(rawSpr)
+		ebitenImage, err := ebiten.NewImageFromImage(rawSpr.ToImage(), ebiten.FilterDefault)
-		ebitenImage, err := ebiten.NewImageFromImage(stdImage, ebiten.FilterDefault)
 		if err != nil {
 			return nil, err
 		}
@@ -54,12 +51,3 @@ func ConvertObject(rawObj *data.Object, name string) (*Object, error) {
 
 	return out, nil
 }
-
-func spriteToImage(sprite *data.Sprite) image.Image {
-	return &image.Paletted{
-		Pix: sprite.Data,
-		Stride: int(sprite.Width),
-		Rect: image.Rect(0, 0, int(sprite.Width), int(sprite.Height)),
-		Palette: data.ColorPalette,
-	}
-}

internal/data/object.go

@@ -3,6 +3,7 @@ package data
 import (
 	"encoding/binary"
 	"fmt"
+	"image"
 	"io"
 	"io/ioutil"
 	"os"
@@ -40,6 +41,16 @@ type Sprite struct {
 	Data []byte
 }
 
+func (s *Sprite) ToImage() *image.Paletted {
+	return &image.Paletted{
+		Pix:     s.Data,
+		Stride:  int(s.Width),
+		Rect:    image.Rect(0, 0, int(s.Width), int(s.Height)),
+		Palette: ColorPalette,
+	}
+}
+
+// dirEntry totals 8 bytes on disk
 type dirEntry struct {
 	Offset uint32 // Offset of the sprite relative to the data block
 	Size   uint32 // Size of the sprite in bytes, including any header
@@ -84,7 +95,7 @@ type Object struct {
 	Sprites  []*Sprite
 }
 
-func LoadObject(filename string) (*Object, error) {
+func LoadObjectLazily(filename string) (*Object, error) {
 	f, err := os.Open(filename)
 	if err != nil {
 		return nil, err
@@ -101,53 +112,85 @@ func LoadObject(filename string) (*Object, error) {
 		return nil, err
 	}
 
-	// Now load all sprites into memory
+	out.Sprites = make([]*Sprite, int(out.NumSprites))
-	dir := make([]dirEntry, out.NumSprites)
-	if _, err := f.Seek(int64(out.DirOffset), io.SeekStart); err != nil {
-		return nil, fmt.Errorf("Seeking to sprite directory: %v", err)
-	}
-
-	if err := binary.Read(f, binary.LittleEndian, &dir); err != nil {
-		return nil, fmt.Errorf("Reading sprite directory: %v", err)
-	}
-
-	if _, err := f.Seek(int64(out.DataOffset), io.SeekStart); err != nil {
-		return nil, fmt.Errorf("Seeking to sprites: %v", err)
-	}
-
-	for i, dirEntry := range dir {
-		if err := dirEntry.Check(); err != nil {
-			return nil, err
-		}
-
-		if _, err := f.Seek(int64(out.DataOffset+dirEntry.Offset), io.SeekStart); err != nil {
-			return nil, fmt.Errorf("Seeking to sprite %v: %v", i, err)
-		}
-
-		sprite := &Sprite{}
-
-		if err := binary.Read(f, binary.LittleEndian, &sprite.SpriteHeader); err != nil {
-			return nil, fmt.Errorf("Reading sprite %v header: %v", i, err)
-		}
-
-		if err := sprite.Check(dirEntry.Size); err != nil {
-			return nil, err
-		}
-
-		buf := io.LimitReader(f, int64(sprite.PixelSize))
-		sprite.Data = make([]byte, int(sprite.Height)*int(sprite.Width))
-
-		// The pixel data is RLE-compressed. Uncompress it here.
-		if err := rle.Expand(buf, sprite.Data); err != nil {
-			return nil, err
-		}
-
-		out.Sprites = append(out.Sprites, sprite)
-	}
 
 	return out, nil
 }
 
+func LoadObject(filename string) (*Object, error) {
+	obj, err := LoadObjectLazily(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := obj.LoadAllSprites(); err != nil {
+		return nil, err
+	}
+
+	return obj, nil
+}
+
+func (o *Object) LoadAllSprites() error {
+	for i := 0; i < int(o.NumSprites); i++ {
+		if err := o.LoadSprite(i); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (o *Object) LoadSprite(idx int) error {
+	if idx < 0 || idx >= int(o.NumSprites) {
+		return fmt.Errorf("Asked for idx %v of %v", idx, o.NumSprites)
+	}
+
+	f, err := os.Open(o.Filename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	var entry dirEntry
+
+	if _, err := f.Seek(int64(o.DirOffset)+int64(idx*8), io.SeekStart); err != nil {
+		return fmt.Errorf("Seeking to sprite directory entry %v: %v", idx, err)
+	}
+
+	if err := binary.Read(f, binary.LittleEndian, &entry); err != nil {
+		return fmt.Errorf("Reading sprite directory entry %v: %v", idx, err)
+	}
+
+	if err := entry.Check(); err != nil {
+		return err
+	}
+
+	if _, err := f.Seek(int64(o.DataOffset+entry.Offset), io.SeekStart); err != nil {
+		return fmt.Errorf("Seeking to sprite %v: %v", idx, err)
+	}
+
+	sprite := &Sprite{}
+
+	if err := binary.Read(f, binary.LittleEndian, &sprite.SpriteHeader); err != nil {
+		return fmt.Errorf("Reading sprite %v header: %v", idx, err)
+	}
+
+	if err := sprite.Check(entry.Size); err != nil {
+		return err
+	}
+
+	buf := io.LimitReader(f, int64(sprite.PixelSize))
+	sprite.Data = make([]byte, int(sprite.Height)*int(sprite.Width))
+
+	// The pixel data is RLE-compressed. Uncompress it here.
+	if err := rle.Expand(buf, sprite.Data); err != nil {
+		return err
+	}
+
+	o.Sprites[idx] = sprite
+	return nil
+}
+
 func LoadObjects(dir string) (map[string]*Object, error) {
 	fis, err := ioutil.ReadDir(dir)
 	if err != nil {

internal/maps/maps.go

@@ -59,13 +59,17 @@ func (h Header) Height() int {
 	return MaxHeight
 }
 
-func (h Header) MapSetFilename() string {
+func (h Header) MapSetName() string {
 	idx := bytes.IndexByte(h.SetName[:], 0)
 	if idx < 0 {
 		idx = 8 // all 8 bytes are used
 	}
 
-	return string(h.SetName[0:idx:idx]) + ".set"
+	return string(h.SetName[0:idx:idx])
+}
+
+func (h Header) MapSetFilename() string {
+	return h.MapSetName() + ".set"
 }
 
 type ObjRef struct {