From 5fccf97f4ba1d35b17e071c1c0a7178a957e711f Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Thu, 19 Mar 2020 22:24:21 +0000 Subject: [PATCH] Lazily load sprite image data This cuts memory use significantly, since many sprites in an object are never used. We can get savings over time by evicting sprites when they go out of scope, but that's, well, out of scope. To achieve this, I introduce an assetstore package that is in charge of loading things from the filesystem. This also allows some lingering case-sensitivity issues to be handled cleanly. I'd hoped that creating fewer ebiten.Image instances would help CPU usage, but that doesn't seem to be the case. --- cmd/view-map/main.go | 111 ++++--------------------- internal/assetstore/assetstore.go | 125 ++++++++++++++++++++++++++++ internal/assetstore/map.go | 89 ++++++++++++++++++++ internal/assetstore/object.go | 64 +++++++++++++++ internal/assetstore/set.go | 50 ++++++++++++ internal/conv/object.go | 14 +--- internal/data/object.go | 131 ++++++++++++++++++++---------- internal/maps/maps.go | 8 +- 8 files changed, 440 insertions(+), 152 deletions(-) create mode 100644 internal/assetstore/assetstore.go create mode 100644 internal/assetstore/map.go create mode 100644 internal/assetstore/object.go create mode 100644 internal/assetstore/set.go diff --git a/cmd/view-map/main.go b/cmd/view-map/main.go index 52bd222..ac4872e 100644 --- a/cmd/view-map/main.go +++ b/cmd/view-map/main.go @@ -2,32 +2,25 @@ package main import ( "flag" - "fmt" "image" "log" "math" "os" - "path/filepath" "github.com/hajimehoshi/ebiten" - "code.ur.gs/lupine/ordoor/internal/conv" - "code.ur.gs/lupine/ordoor/internal/data" - "code.ur.gs/lupine/ordoor/internal/maps" - "code.ur.gs/lupine/ordoor/internal/sets" + "code.ur.gs/lupine/ordoor/internal/assetstore" "code.ur.gs/lupine/ordoor/internal/ui" ) var ( gamePath = flag.String("game-path", "./orig", "Path to a WH40K: Chaos Gate installation") - mapFile = flag.String("map", "", "Prefix path to a .map file, e.g. ./orig/Maps/Chapter01.MAP") - txtFile = flag.String("txt", "", "Prefix path to a .txt file, e.g. ./orig/Maps/Chapter01.txt") + gameMap = flag.String("map", "", "Name of a map, e.g., Chapter01") ) type env struct { - gameMap *maps.GameMap - set *sets.MapSet - objects map[string]*conv.Object + assets *assetstore.AssetStore + area *assetstore.Map step int state state @@ -43,39 +36,19 @@ type state struct { func main() { flag.Parse() - if *gamePath == "" || *mapFile == "" || *txtFile == "" { + if *gamePath == "" || *gameMap == "" { flag.Usage() os.Exit(1) } - gameMap, err := maps.LoadGameMapByFiles(*mapFile, *txtFile) + assets, err := assetstore.New(*gamePath) if err != nil { - log.Fatalf("Couldn't load map file: %v", err) + log.Fatalf("Failed to scan root directory %v: %v", *gamePath, err) } - setFile := filepath.Join(*gamePath, "Sets", gameMap.MapSetFilename()) - log.Println(setFile) - mapSet, err := sets.LoadSet(setFile) + area, err := assets.Map(*gameMap) if err != nil { - log.Fatalf("Couldn't load set file %s: %v", setFile, err) - } - - objects := make([]*conv.Object, 0, len(mapSet.Palette)) - for _, name := range mapSet.Palette { - objFile := filepath.Join(*gamePath, "Obj", name+".obj") - rawObj, err := data.LoadObject(objFile) - if err != nil { - log.Fatalf("Failed to load %s: %v", name, err) - } - - rawObj.Name = name - - obj, err := conv.ConvertObject(rawObj, name) - if err != nil { - log.Fatal(err) - } - - objects = append(objects, obj) + log.Fatalf("Failed to load map %v: %v", *gameMap, err) } state := state{ @@ -83,14 +56,13 @@ func main() { origin: image.Point{0, 3000}, // FIXME: haxxx } env := &env{ - gameMap: gameMap, - set: mapSet, - objects: conv.MapByName(objects), + area: area, + assets: assets, state: state, lastState: state, } - win, err := ui.NewWindow("View Map " + *mapFile) + win, err := ui.NewWindow("View Map " + *gameMap) if err != nil { log.Fatal("Couldn't create window: %v", err) } @@ -123,30 +95,6 @@ func (e *env) Update() error { return nil } -func (e *env) getSprite(palette []string, ref maps.ObjRef) (*conv.Sprite, error) { - // There seems to be an active bit that hides many sins - if !ref.IsActive() { - return nil, nil - } - - if ref.Index() >= len(palette) { - return nil, fmt.Errorf("Palette too small: %v requested", ref.Index()) - } - - name := palette[ref.Index()] - - obj := e.objects[name] - if obj == nil { - return nil, fmt.Errorf("Failed to find surface sprite %#v -> %q", ref, name) - } - - if ref.Sprite() >= len(obj.Sprites) { - return nil, fmt.Errorf("Out-of-index sprite %v requested for %v", ref.Sprite(), name) - } - - return obj.Sprites[ref.Sprite()], nil -} - func (e *env) Draw(screen *ebiten.Image) error { // Bounds clipping // http://www.java-gaming.org/index.php?topic=24922.0 @@ -177,8 +125,7 @@ func (e *env) Draw(screen *ebiten.Image) error { x := (a + b) / 2 y := (a - b) / 2 - if x < int(e.gameMap.MinWidth) || x >= int(e.gameMap.MaxWidth) || - y < int(e.gameMap.MinLength) || y >= int(e.gameMap.MaxLength) { + if !image.Pt(x, y).In(e.area.Rect) { continue } @@ -192,31 +139,9 @@ func (e *env) Draw(screen *ebiten.Image) error { } func (e *env) renderCell(x, y, z int, screen *ebiten.Image) error { - var sprites []*conv.Sprite - cell := e.gameMap.Cells.At(x, y, z) - - if spr, err := e.getSprite(e.set.Palette, cell.Surface); err != nil { - log.Printf("%v %v %v surface: %v", x, y, z, err) - } else if spr != nil { - sprites = append(sprites, spr) - } - - if spr, err := e.getSprite(e.set.Palette, cell.Center); err != nil { - log.Printf("%v %v %v center: %v", x, y, z, err) - } else if spr != nil { - sprites = append(sprites, spr) - } - - if spr, err := e.getSprite(e.set.Palette, cell.Left); err != nil { - log.Printf("%v %v %v left: %v", x, y, z, err) - } else if spr != nil { - sprites = append(sprites, spr) - } - - if spr, err := e.getSprite(e.set.Palette, cell.Right); err != nil { - log.Printf("%v %v %v right: %v", x, y, z, err) - } else if spr != nil { - sprites = append(sprites, spr) + images, err := e.area.ImagesForCell(x, y, z) + if err != nil { + return err } iso := ebiten.GeoM{} @@ -231,8 +156,8 @@ func (e *env) renderCell(x, y, z int, screen *ebiten.Image) error { // TODO: iso.Scale(e.state.zoom, e.state.zoom) // apply current zoom factor - for _, sprite := range sprites { - if err := screen.DrawImage(sprite.Image, &ebiten.DrawImageOptions{GeoM: iso}); err != nil { + for _, img := range images { + if err := screen.DrawImage(img, &ebiten.DrawImageOptions{GeoM: iso}); err != nil { return err } } diff --git a/internal/assetstore/assetstore.go b/internal/assetstore/assetstore.go new file mode 100644 index 0000000..2d96f31 --- /dev/null +++ b/internal/assetstore/assetstore.go @@ -0,0 +1,125 @@ +package assetstore + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +const ( + RootDir = "" // Used in the entryMap for entries pertaining to the root dir +) + +type entryMap map[string]map[string]string + +// type AssetStore is responsible for lazily loading game data when it is +// required. Applications shouldn't need to do anything except set one of these +// up, pointing at the game dir root, to access all assets. +// +// Assets should be loaded on-demand to keep memory costs as low as possible. +// Cross-platform differences such as filename case sensitivity are also dealt +// with here. +// +// We assume the directory is read-only. You can run Refresh() if you make a +// change. +type AssetStore struct { + RootDir string + + // Case-insensitive file lookup. + // {"":{"anim":"Anim", "obj":"Obj", ...}, "anim":{ "warhammer.ani":"WarHammer.ani" }, ...} + entries entryMap + + // These members are used to store things we've already loaded + maps map[string]*Map + objs map[string]*Object + sets map[string]*Set +} + +// New returns a new AssetStore +func New(dir string) (*AssetStore, error) { + store := &AssetStore{ + RootDir: dir, + } + + // fill entryMap + if err := store.Refresh(); err != nil { + return nil, err + } + + return store, nil +} + +func (a *AssetStore) Refresh() error { + rootEntries, err := processDir(a.RootDir) + if err != nil { + return fmt.Errorf("failed to process %v: %v", a.RootDir, err) + } + + newEntryMap := make(entryMap, len(rootEntries)) + newEntryMap[RootDir] = rootEntries + + for lower, natural := range rootEntries { + path := filepath.Join(a.RootDir, natural) + fi, err := os.Stat(path) + if err != nil { + return fmt.Errorf("Failed to stat %v: %v", path, err) + } + + if fi.IsDir() { + entries, err := processDir(path) + if err != nil { + return fmt.Errorf("Failed to process %v: %v", path, err) + } + + newEntryMap[lower] = entries + } + } + + // Refresh + a.entries = newEntryMap + a.maps = make(map[string]*Map) + a.objs = make(map[string]*Object) + a.sets = make(map[string]*Set) + + return nil +} + +func (a *AssetStore) lookup(name, ext string, dirs ...string) (string, error) { + filename := canonical(name + "." + ext) + + for _, dir := range dirs { + dir = canonical(dir) + if base, ok := a.entries[dir]; ok { + if file, ok := base[filename]; ok { + actualDir := a.entries[RootDir][dir] + return filepath.Join(a.RootDir, actualDir, file), nil + } + } + } + + return "", os.ErrNotExist +} + +func canonical(s string) string { + return strings.ToLower(s) +} + +func processDir(dir string) (map[string]string, error) { + entries, err := ioutil.ReadDir(dir) + if err != nil { + return nil, err + } + + out := make(map[string]string, len(entries)) + for _, entry := range entries { + if entry.Name() == "." || entry.Name() == ".." { + continue + } + + out[canonical(entry.Name())] = entry.Name() + } + + return out, nil +} diff --git a/internal/assetstore/map.go b/internal/assetstore/map.go new file mode 100644 index 0000000..10d954c --- /dev/null +++ b/internal/assetstore/map.go @@ -0,0 +1,89 @@ +package assetstore + +import ( + "github.com/hajimehoshi/ebiten" + "image" + + "code.ur.gs/lupine/ordoor/internal/maps" +) + +type Map struct { + assets *AssetStore + set *Set + Rect image.Rectangle + + raw *maps.GameMap +} + +// Map loads a game map with the given name (e.g. "Chapter01") +func (a *AssetStore) Map(name string) (*Map, error) { + name = canonical(name) + + if m, ok := a.maps[name]; ok { + return m, nil + } + + mapFile, err := a.lookup(name, "map", "Maps", "MultiMaps") + if err != nil { + return nil, err + } + + txtFile, err := a.lookup(name, "txt", "Maps", "MultiMaps") + if err != nil { + return nil, err + } + + raw, err := maps.LoadGameMapByFiles(mapFile, txtFile) + if err != nil { + return nil, err + } + + // The set for a map is small and frequently referenced, so load it here + set, err := a.Set(raw.MapSetName()) + if err != nil { + return nil, err + } + + m := &Map{ + Rect: image.Rect( + int(raw.MinWidth), + int(raw.MinLength), + int(raw.MaxWidth), + int(raw.MaxLength), + ), + assets: a, + raw: raw, + set: set, + } + + a.maps[canonical(name)] = m + + return m, nil +} + +// ImagesForCell returns the sprites needed to correctly render this cell. +// They should be rendered from first to last to get the correct ordering +func (m *Map) ImagesForCell(x, y, z int) ([]*ebiten.Image, error) { + cell := m.raw.At(x, y, z) + images := make([]*ebiten.Image, 0, 4) + + for _, ref := range []maps.ObjRef{cell.Surface, cell.Center, cell.Left, cell.Right} { + if !ref.IsActive() { + continue + } + + obj, err := m.set.Object(ref.Index()) + if err != nil { + return nil, err + } + + img, err := obj.Image(ref.Sprite()) + if err != nil { + return nil, err + } + + images = append(images, img) + } + + return images, nil +} diff --git a/internal/assetstore/object.go b/internal/assetstore/object.go new file mode 100644 index 0000000..e6d20c8 --- /dev/null +++ b/internal/assetstore/object.go @@ -0,0 +1,64 @@ +package assetstore + +import ( + "github.com/hajimehoshi/ebiten" + + "code.ur.gs/lupine/ordoor/internal/data" +) + +type Object struct { + assets *AssetStore + images []*ebiten.Image + + raw *data.Object +} + +func (a *AssetStore) Object(name string) (*Object, error) { + name = canonical(name) + + if obj, ok := a.objs[name]; ok { + return obj, nil + } + + filename, err := a.lookup(name, "obj", "Obj") + if err != nil { + return nil, err + } + + raw, err := data.LoadObjectLazily(filename) + if err != nil { + return nil, err + } + + obj := &Object{ + assets: a, + images: make([]*ebiten.Image, int(raw.NumSprites)), + raw: raw, + } + a.objs[name] = obj + + return obj, nil +} + +// Filled lazily +func (o *Object) Image(idx int) (*ebiten.Image, error) { + if img := o.images[idx]; img != nil { + return img, nil + } + + if o.raw.Sprites[idx] == nil { + if err := o.raw.LoadSprite(idx); err != nil { + return nil, err + } + } + + stdImg := o.raw.Sprites[idx].ToImage() + img, err := ebiten.NewImageFromImage(stdImg, ebiten.FilterDefault) + if err != nil { + return nil, err + } + + o.images[idx] = img + + return img, nil +} diff --git a/internal/assetstore/set.go b/internal/assetstore/set.go new file mode 100644 index 0000000..5fd0090 --- /dev/null +++ b/internal/assetstore/set.go @@ -0,0 +1,50 @@ +package assetstore + +import ( + "errors" + + "code.ur.gs/lupine/ordoor/internal/sets" +) + +var ( + ErrOutOfBounds = errors.New("Out of bounds") +) + +type Set struct { + assets *AssetStore + + raw *sets.MapSet +} + +func (s *Set) Object(idx int) (*Object, error) { + if idx < 0 || idx >= len(s.raw.Palette) { + return nil, ErrOutOfBounds + } + + return s.assets.Object(s.raw.Palette[idx]) +} + +func (a *AssetStore) Set(name string) (*Set, error) { + name = canonical(name) + if set, ok := a.sets[name]; ok { + return set, nil + } + + filename, err := a.lookup(name, "set", "Sets") + if err != nil { + return nil, err + } + + raw, err := sets.LoadSet(filename) + if err != nil { + return nil, err + } + + set := &Set{ + assets: a, + raw: raw, + } + a.sets[name] = set + + return set, nil +} diff --git a/internal/conv/object.go b/internal/conv/object.go index 7d831b2..0a04e68 100644 --- a/internal/conv/object.go +++ b/internal/conv/object.go @@ -1,8 +1,6 @@ package conv import ( - "image" - "github.com/hajimehoshi/ebiten" "code.ur.gs/lupine/ordoor/internal/data" @@ -43,8 +41,7 @@ func ConvertObject(rawObj *data.Object, name string) (*Object, error) { for i, rawSpr := range rawObj.Sprites { w := int(rawSpr.Width) h := int(rawSpr.Height) - stdImage := spriteToImage(rawSpr) - ebitenImage, err := ebiten.NewImageFromImage(stdImage, ebiten.FilterDefault) + ebitenImage, err := ebiten.NewImageFromImage(rawSpr.ToImage(), ebiten.FilterDefault) if err != nil { return nil, err } @@ -54,12 +51,3 @@ func ConvertObject(rawObj *data.Object, name string) (*Object, error) { return out, nil } - -func spriteToImage(sprite *data.Sprite) image.Image { - return &image.Paletted{ - Pix: sprite.Data, - Stride: int(sprite.Width), - Rect: image.Rect(0, 0, int(sprite.Width), int(sprite.Height)), - Palette: data.ColorPalette, - } -} diff --git a/internal/data/object.go b/internal/data/object.go index 724283c..9b730be 100644 --- a/internal/data/object.go +++ b/internal/data/object.go @@ -3,6 +3,7 @@ package data import ( "encoding/binary" "fmt" + "image" "io" "io/ioutil" "os" @@ -40,6 +41,16 @@ type Sprite struct { Data []byte } +func (s *Sprite) ToImage() *image.Paletted { + return &image.Paletted{ + Pix: s.Data, + Stride: int(s.Width), + Rect: image.Rect(0, 0, int(s.Width), int(s.Height)), + Palette: ColorPalette, + } +} + +// dirEntry totals 8 bytes on disk type dirEntry struct { Offset uint32 // Offset of the sprite relative to the data block Size uint32 // Size of the sprite in bytes, including any header @@ -84,7 +95,7 @@ type Object struct { Sprites []*Sprite } -func LoadObject(filename string) (*Object, error) { +func LoadObjectLazily(filename string) (*Object, error) { f, err := os.Open(filename) if err != nil { return nil, err @@ -101,53 +112,85 @@ func LoadObject(filename string) (*Object, error) { return nil, err } - // Now load all sprites into memory - dir := make([]dirEntry, out.NumSprites) - if _, err := f.Seek(int64(out.DirOffset), io.SeekStart); err != nil { - return nil, fmt.Errorf("Seeking to sprite directory: %v", err) - } - - if err := binary.Read(f, binary.LittleEndian, &dir); err != nil { - return nil, fmt.Errorf("Reading sprite directory: %v", err) - } - - if _, err := f.Seek(int64(out.DataOffset), io.SeekStart); err != nil { - return nil, fmt.Errorf("Seeking to sprites: %v", err) - } - - for i, dirEntry := range dir { - if err := dirEntry.Check(); err != nil { - return nil, err - } - - if _, err := f.Seek(int64(out.DataOffset+dirEntry.Offset), io.SeekStart); err != nil { - return nil, fmt.Errorf("Seeking to sprite %v: %v", i, err) - } - - sprite := &Sprite{} - - if err := binary.Read(f, binary.LittleEndian, &sprite.SpriteHeader); err != nil { - return nil, fmt.Errorf("Reading sprite %v header: %v", i, err) - } - - if err := sprite.Check(dirEntry.Size); err != nil { - return nil, err - } - - buf := io.LimitReader(f, int64(sprite.PixelSize)) - sprite.Data = make([]byte, int(sprite.Height)*int(sprite.Width)) - - // The pixel data is RLE-compressed. Uncompress it here. - if err := rle.Expand(buf, sprite.Data); err != nil { - return nil, err - } - - out.Sprites = append(out.Sprites, sprite) - } + out.Sprites = make([]*Sprite, int(out.NumSprites)) return out, nil } +func LoadObject(filename string) (*Object, error) { + obj, err := LoadObjectLazily(filename) + if err != nil { + return nil, err + } + + if err := obj.LoadAllSprites(); err != nil { + return nil, err + } + + return obj, nil +} + +func (o *Object) LoadAllSprites() error { + for i := 0; i < int(o.NumSprites); i++ { + if err := o.LoadSprite(i); err != nil { + return err + } + } + + return nil +} + +func (o *Object) LoadSprite(idx int) error { + if idx < 0 || idx >= int(o.NumSprites) { + return fmt.Errorf("Asked for idx %v of %v", idx, o.NumSprites) + } + + f, err := os.Open(o.Filename) + if err != nil { + return err + } + defer f.Close() + + var entry dirEntry + + if _, err := f.Seek(int64(o.DirOffset)+int64(idx*8), io.SeekStart); err != nil { + return fmt.Errorf("Seeking to sprite directory entry %v: %v", idx, err) + } + + if err := binary.Read(f, binary.LittleEndian, &entry); err != nil { + return fmt.Errorf("Reading sprite directory entry %v: %v", idx, err) + } + + if err := entry.Check(); err != nil { + return err + } + + if _, err := f.Seek(int64(o.DataOffset+entry.Offset), io.SeekStart); err != nil { + return fmt.Errorf("Seeking to sprite %v: %v", idx, err) + } + + sprite := &Sprite{} + + if err := binary.Read(f, binary.LittleEndian, &sprite.SpriteHeader); err != nil { + return fmt.Errorf("Reading sprite %v header: %v", idx, err) + } + + if err := sprite.Check(entry.Size); err != nil { + return err + } + + buf := io.LimitReader(f, int64(sprite.PixelSize)) + sprite.Data = make([]byte, int(sprite.Height)*int(sprite.Width)) + + // The pixel data is RLE-compressed. Uncompress it here. + if err := rle.Expand(buf, sprite.Data); err != nil { + return err + } + + o.Sprites[idx] = sprite + return nil +} + func LoadObjects(dir string) (map[string]*Object, error) { fis, err := ioutil.ReadDir(dir) if err != nil { diff --git a/internal/maps/maps.go b/internal/maps/maps.go index f28bfc5..46c9598 100644 --- a/internal/maps/maps.go +++ b/internal/maps/maps.go @@ -59,13 +59,17 @@ func (h Header) Height() int { return MaxHeight } -func (h Header) MapSetFilename() string { +func (h Header) MapSetName() string { idx := bytes.IndexByte(h.SetName[:], 0) if idx < 0 { idx = 8 // all 8 bytes are used } - return string(h.SetName[0:idx:idx]) + ".set" + return string(h.SetName[0:idx:idx]) +} + +func (h Header) MapSetFilename() string { + return h.MapSetName() + ".set" } type ObjRef struct {