Convert from BoltDB to Maildir storage for emails

internal/store/account.go

@@ -29,9 +29,6 @@ type Account struct {
 
 	// As generated by HashPassword
 	PasswordHash string
-
-	// Where to put messages by default. FK: mailbox.id
-	DefaultMailbox uint64
 }
 
 // HashPassword turns a plaintext password into a crypt()ed string, using bcrypt

internal/store/maildir.go

@@ -2,42 +2,123 @@ package store
 
 import (
 	"errors"
+	"net/mail"
+	"os"
+	"path/filepath"
+
+	"github.com/luksen/maildir"
+
+	"ur.gs/crockery/internal/imap/uidlist"
 )
 
-type MailboxInterface interface {
-	CreateMailbox(*Mailbox) error
-	FindMailbox(account Account, name string) (Mailbox, error)
-	FindMailboxes(account Account) ([]Mailbox, error)
+type MaildirInterface interface {
+	CreateMaildir(*Maildir) error
+	FindMaildir(account Account, name string) (Maildir, error)
+	FindMaildirs(account Account) ([]Maildir, error)
+	FindMessagesInRange(m Maildir, start, end uint64) ([]Message, error)
 }
 
-type Mailbox struct {
-	ID      uint64 `storm:"id,increment"`
-	Account string `storm:"index"` // FK accounts.username
-	Name    string `storm:"index"`
+type Maildir struct {
+	Account Account
+	Name    string
+
+	directory string
+
+	filesystem maildir.Dir
+	uids       *uidlist.List
 }
 
-func (c *concrete) CreateMailbox(mailbox *Mailbox) error {
-	return c.storm.Save(mailbox)
+func (m *Maildir) Rel() string {
+	return filepath.Join(m.Account.Username, m.Name)
 }
 
-func (c *concrete) FindMailbox(account Account, name string) (Mailbox, error) {
-	// FIXME I don't know how to storm
-	candidates, err := c.FindMailboxes(account)
+func (m *Maildir) Directory() string {
+	return m.directory
+}
+
+func (m *Maildir) Header(key string) (mail.Header, error) {
+	return m.filesystem.Header(key)
+}
+
+func (c *concrete) CreateMaildir(m *Maildir) error {
+	if m.directory == "" {
+		m.directory = c.buildMaildirPath(m.Account, m.Name)
+	}
+
+	if string(m.filesystem) == "" {
+		m.filesystem = maildir.Dir(m.directory)
+	}
+
+	err := m.filesystem.Create()
 	if err != nil {
-		return Mailbox{}, err
+		return err
 	}
 
-	for _, mbox := range candidates {
-		if mbox.Name == name {
-			return mbox, nil
+	if m.uids == nil {
+		uids, err := uidlist.Create(m.directory)
+		if err != nil {
+			// FIXME: this leaves a dangling Maildir
+			return err
 		}
+
+		m.uids = uids
 	}
 
-	return Mailbox{}, errors.New("not found")
+	return nil
 }
 
-func (c *concrete) FindMailboxes(account Account) ([]Mailbox, error) {
-	var out []Mailbox
+func (c *concrete) FindMaildir(account Account, name string) (Maildir, error) {
+	// FIXME: multiple IMAP sessions get different Maildir instances, so breaking
+	// any hope at locking (sigh)
+	m := Maildir{
+		Account:   account,
+		Name:      name,
+		directory: c.buildMaildirPath(account, name),
+	}
+	m.filesystem = maildir.Dir(m.directory)
 
-	return out, c.storm.Find("Account", account.Username, &out)
+	if _, err := os.Stat(m.Directory()); os.IsNotExist(err) {
+		return Maildir{}, errors.New("not found")
+	}
+
+	uids, err := uidlist.Open(m.directory)
+	if err != nil {
+		return Maildir{}, err
+	}
+	m.uids = uids
+
+	return m, nil
+}
+
+func (c *concrete) FindMaildirs(account Account) ([]Maildir, error) {
+	// TODO: iterate through the maildir looking for submaildirs
+	defaultMaildir, err := c.FindMaildir(account, "")
+	if err != nil {
+		return nil, err
+	}
+
+	return []Maildir{defaultMaildir}, nil
+}
+
+func (c *concrete) buildMaildirPath(account Account, parts ...string) string {
+	bits := append([]string{BuildMailboxesPath(c.home), account.Username}, parts...)
+
+	return filepath.Join(bits...)
+}
+
+func (c *concrete) FindMessagesInRange(m Maildir, start, end uint64) ([]Message, error) {
+	var out []Message
+	entries := m.uids.EntriesInRange(start, end)
+
+	for _, entry := range entries {
+		msg := Message{
+			UID:     entry.UID,
+			Maildir: m,
+			Key:     entry.Key(),
+		}
+
+		out = append(out, msg)
+	}
+
+	return out, nil
 }

internal/store/message.go

@@ -1,50 +1,108 @@
 package store
 
 import (
+	"fmt"
+	"io"
+	"log"
 	"net/mail"
-
-	"github.com/asdine/storm/q"
+	"path/filepath"
+	"strings"
 )
 
+type Message struct {
+	Maildir Maildir // maildir.name
+
+	UID      uint64
+	Key      string
+	Data     io.ReadCloser
+	hdrCache mail.Header
+}
+
+// TODO: the cache is probably fine
+func (m *Message) Header() (mail.Header, error) {
+	if m.hdrCache != nil {
+		return m.hdrCache, nil
+	}
+
+	hdr, err := m.Maildir.Header(m.Key)
+	if err != nil {
+		return nil, err
+	}
+
+	m.hdrCache = hdr
+	return hdr, nil
+}
+
+func (m *Message) Len() uint64 {
+	return 0 // FIXME
+}
+
+func (m *Message) HeaderString() (string, error) {
+	var sb strings.Builder
+
+	hdr, err := m.Header()
+	if err != nil {
+		return "", err
+	}
+
+	for k, vs := range hdr {
+		for _, v := range vs {
+			fmt.Fprintf(&sb, "%s: %s\r\n", k, v)
+		}
+	}
+	return sb.String(), nil // FIXME
+}
+
 type MessageInterface interface {
 	CreateMessage(Message) error
-	FindMessages(Mailbox) ([]Message, error)
-
-	// if end == 0, it means "from start to most recent"
-	FindMessagesInRange(mailbox Mailbox, start, end uint64) ([]Message, error)
-}
-
-type Message struct {
-	// FIXME: this should be per-mailbox, IMAP "only" allows 32-bit message UIDs?
-	ID uint64 `storm:"id,increment"`
-
-	Mailbox uint64 `storm:"index"` // FK mailbox.id
-
-	Header mail.Header // The parsed headers
-	EML    []byte      // The full email, unaltered
 }
 
+// DeliverMessage takes the given message and delivers it to the correct maildir
+// It does not call Close() on message.Data
 func (c *concrete) CreateMessage(message Message) error {
-	return c.storm.Save(&message)
-}
+	// TODO: recover from panics, assure Abort() is called for them
+	maildir := message.Maildir
 
-func (c *concrete) FindMessages(mailbox Mailbox) ([]Message, error) {
-	var out []Message
-
-	return out, c.storm.Find("Mailbox", mailbox.ID, &out)
-}
-
-func (c *concrete) FindMessagesInRange(mailbox Mailbox, start, end uint64) ([]Message, error) {
-	var out []Message
-
-	matchers := []q.Matcher{
-		q.Eq("Mailbox", mailbox.ID),
-		q.Gte("ID", start),
+	delivery, err := maildir.filesystem.NewDelivery()
+	if err != nil {
+		return err
 	}
 
-	if end > 0 {
-		matchers = append(matchers, q.Lte("ID", end))
+	if _, err := io.Copy(delivery, message.Data); err != nil {
+		delivery.Abort()
+		return err
 	}
 
-	return out, c.storm.Select(matchers...).Find(&out)
+	if err := delivery.Close(); err != nil {
+		delivery.Abort()
+		return err
+	}
+
+	// FIXME: this moves files in the maildir before the dovecot-uidlist is
+	// updated. Not ideal.
+	// FIXME: we're not
+	keys, err := maildir.filesystem.Unseen()
+	if err != nil {
+		log.Printf("Failed moving messages from new/ to cur/: %v", err)
+	}
+
+	var filenames []string
+	for _, key := range keys {
+		filename, err := maildir.filesystem.Filename(key)
+		if err != nil {
+			log.Printf("Failed getting filename for unseen message %q: %v", key, err)
+			continue
+		}
+		filenames = append(filenames, filepath.Base(filename))
+	}
+
+	// Don't bubble the error up here as delivery has succeeded, IMAP just
+	// doesn't know it has. We don't want the client to attempt redelivery.
+	//
+	// FIXME: this is IMAP stuff that needs moving
+	if err := maildir.uids.Append(filenames...); err != nil {
+		log.Printf("Allocating UIDs for unseen messages failed: %v", err)
+	}
+
+	return nil
 }

internal/store/spool.go

@@ -5,40 +5,47 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
-	"net/mail"
+	//	"net/mail"
+
+	"golang.org/x/sync/errgroup"
 )
 
 type SpoolInterface interface {
 	SpoolMessage([]Account, io.Reader) error
 }
 
-// FIXME: we don't actually spool for now, we just store it to each user's
-// mailbox. This might lead to oddness in the partial delivery case.
+func (c *concrete) deliver(account Account, data []byte) error {
+	// FIXME: this could be a FindOrCreateMaildir call for atomic create?
+	maildir, err := c.FindMaildir(account, "")
+	if err != nil {
+		return err
+	}
+
+	message := Message{
+		Maildir: maildir,
+		Data:    ioutil.NopCloser(bytes.NewReader(data)),
+	}
+
+	log.Printf("Creating email for %#v", account)
+	log.Print(string(data))
+
+	return c.CreateMessage(message)
+}
+
+// FIXME: we don't actually spool for now, we just store it in each user's
+// maildir. This might lead to oddness in the partial delivery case.
 func (c *concrete) SpoolMessage(recipients []Account, r io.Reader) error {
 	data, err := ioutil.ReadAll(r)
 	if err != nil {
 		return err
 	}
 
-	message, err := mail.ReadMessage(bytes.NewReader(data))
-	if err != nil {
-		return err
-	}
+	var g errgroup.Group
 
 	for _, recipient := range recipients {
-		message := Message{
-			Mailbox: recipient.DefaultMailbox,
-			Header:  message.Header,
-			EML:     data,
-		}
-
-		log.Printf("Creating email: %#v", message)
-		log.Print(string(data))
-
-		if err := c.CreateMessage(message); err != nil {
-			return err
-		}
+		recipient := recipient
+		g.Go(func() error { return c.deliver(recipient, data) })
 	}
 
-	return nil
+	return g.Wait()
 }

internal/store/store.go

@@ -5,11 +5,18 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io"
+	"os"
+	"path/filepath"
 
 	"github.com/asdine/storm"
 	"github.com/coreos/bbolt"
 )
 
+const (
+	DatabaseFilename = "crockery.db"
+	MailboxesDirname = "mailboxes"
+)
+
 type Interface interface {
 	Domain() string
 	TLS() tls.Certificate
@@ -19,26 +26,42 @@ type Interface interface {
 	SetTLS([]byte, []byte) error
 
 	AccountInterface
-	MailboxInterface
+	MaildirInterface
 	MessageInterface
 	SpoolInterface
 
 	io.Closer
 }
 
+func buildPath(parts ...string) string {
+	if len(parts) == 0 || parts[0] == "" {
+		return ""
+	}
+
+	return filepath.Join(parts...)
+}
+func BuildDatabasePath(home string) string {
+	return buildPath(home, DatabaseFilename)
+}
+
+func BuildMailboxesPath(home string) string {
+	return buildPath(home, MailboxesDirname)
+}
+
 func IsNotFound(err error) bool {
 	return err.Error() == "not found" // Magic hardcoded value in storm
 }
 
-func New(ctx context.Context, filename string) (Interface, error) {
+func New(ctx context.Context, home string) (Interface, error) {
+	filename := BuildDatabasePath(home)
 	db, err := storm.Open(filename, storm.BoltOptions(0600, &bolt.Options{}))
 	if err != nil {
 		return nil, err
 	}
 
 	out := &concrete{
-		filename: filename,
-		storm:    db,
+		home:  home,
+		storm: db,
 	}
 
 	if err := out.setup(); err != nil {
@@ -48,19 +71,30 @@ func New(ctx context.Context, filename string) (Interface, error) {
 	return out, nil
 }
 
-func Init(filename, domain string, certPEM, keyPEM []byte) error {
+func Init(home, domain string, certPEM, keyPEM []byte) error {
+	filename := BuildDatabasePath(home)
+	mailboxes := BuildMailboxesPath(home)
+
+	if _, err := os.Stat(mailboxes); !os.IsNotExist(err) {
+		return fmt.Errorf("mailboxes directory %q already exists", mailboxes)
+	}
+
 	db, err := storm.Open(filename, storm.BoltOptions(0600, &bolt.Options{}))
 	if err != nil {
 		return err
 	}
 
 	builder := &concrete{
-		filename: filename,
-		storm:    db,
+		home:  home,
+		storm: db,
 	}
 
 	defer builder.Close()
 
+	if err := os.Mkdir(mailboxes, 0700); err != nil {
+		return fmt.Errorf("Couldn't create mailboxes directory %q: %v", mailboxes, err)
+	}
+
 	if err := builder.SetDomain(domain); err != nil {
 		return fmt.Errorf("Couldn't set domain: %v", err)
 	}
@@ -73,8 +107,9 @@ func Init(filename, domain string, certPEM, keyPEM []byte) error {
 }
 
 type concrete struct {
-	filename     string
-	storm        *storm.DB
+	home  string
+	storm *storm.DB
+
 	domainBucket storm.Node
 
 	// These are persisted in BoltDB, but we might as well keep them in memory