diff --git a/DESIGN.md b/DESIGN.md index 6e7edb7..6ecda54 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -40,19 +40,9 @@ following, though: * `Admin` * `Wildcard` * `PasswordHash` - * `Message` - * `ID` - * `Username` - * `Mailbox` - * `Header` - * `Body` I don't seem to be able to get accounts for a list of usernames very easily, or indexed-ly. -The message body is being stored inefficiently (json []byte, so base64-encoded) - -Message username and mailbox are indexed, but is that good enough? Perhaps we -should have a bucket per mailbox or something? - -TODO: rewrite this (and the code) to make Maildir into a thing +Messages are stored in a traditional Maildir structure structure following a +`/mailboxes//` hierarchy. diff --git a/Makefile b/Makefile index db6a9aa..055fdc2 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ SOURCES := $(shell find $(SOURCEDIR) -iname *.go) crockery: $(SOURCES) - go build ur.gs/crockery/cmd/crockery && sudo setcap 'cap_net_bind_service=+ep' crockery + # FIXME: with CGO enabled, setcap creates a binary that segfaults??? + CGO_ENABLED=0 go build ur.gs/crockery/cmd/crockery + sudo setcap 'cap_net_bind_service=+ep' ./crockery clean: rm -f crockery diff --git a/README.md b/README.md index 04c4953..9cccae4 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,9 @@ ## What -* SMTP + IMAP email, plus bundled HTTP webmail client -* Devoted to the task of serving email for a single domain +* Email! +* Devoted to the task of being a vertically-integrated email solution for a single domain +* SMTP+IMAP email, autodiscovery, and bundled HTTP(S) webmail client * Can get its own valid TLS certificates via LE, or use supplied ones * Built-in anti-spam & anti-virus measures * Built-in DKIM support @@ -13,7 +14,7 @@ * A "master" wildcard account (if desired) * Simple import from GMail, etc * Simple export to GMail, etc -* Nice fast search +* Nice, fast search * Interact with crockery with simple, natural-ish emails to do things * SMTP TLS verification - enabled by default * Try to email foo@shady.net - verification fails, email marked pending, fail in 7 days @@ -24,6 +25,7 @@ * Send just this email * Whitelist shady.net (pins certificate) * What other interactions might we like? + * Use HTTPS links instead of emails? ## What not @@ -83,10 +85,32 @@ needs (25, 587, 149, 993) **without** running as root. Don't bother with it if you're going to be running crockery as root, e.g., as a container or a single- purpose system. +You may also provide these capabilities via systemd by using lines like these in +the `[Service]` definition: + +```ini +CapabilityBoundingSet=CAP_NET_BIND_SERVICE +AmbientCapabilities=CAP_NET_BIND_SERVICE + +NoNewPrivileges=true +``` + +If you're doing this, you may also want to set: + +```ini +PrivateTmp=true +PrivateDevices=true +ProtectHome=true +ProtectSystem=full +ReadWriteDirectories=/var/lib/caddy +``` + +TODO: write up a .service file that can be dropped in + ### Initialize a new database ``` -crockery init \ +$ crockery init \ --domain \ --cert \ --key \ @@ -96,13 +120,23 @@ crockery init \ You can also provide the postmaster password in an environment variable: ``` -CROCKERY_POSTMASTER_PASSWORD="" crockery init --domain ---cert --key +$ CROCKERY_POSTMASTER_PASSWORD="" crockery init \ + --domain \ + --cert \ + --key ``` -You can provide a custom database name with `--db `: +By default, crockery will initialize the current user's home directory. You can +provide a custom directory to work with by specifying `--home`, e.g.: ``` -crockery --db foo.db init --domain ---cert --key +$ crockery \ + --home \ + init \ + --domain \ + --cert \ + --key \ + --postmaster-password ``` ### Run the server @@ -111,9 +145,10 @@ crockery --db foo.db init --domain ---cert --key ` if the default of `./crockery.db` doesn't -suit. Crockery will load the configuration from the database and begin serving -mail based on it. Received emails are also stored in the same file. +Again, you can use `--home ` if the default of `$HOME` doesn't suit. +Crockery will load the configuration from the database and begin serving mail +based on it. Received emails are stored in a mailbox determined by the username, +to be found in the `/mailboxes` directory ### Configuration @@ -124,7 +159,7 @@ apart from `init` as detailed above. Some ideas: ``` $ crockery change x y z (domain, user password, etc) -$ crockery reindex (throw away existing indexes, regenerate) +$ crockery reindex [account] (throw away existing indexes, regenerate) $ crockery whitelist tls (allow the domain to be sent to with insecure/no TLS) $ crockery whitelist receipt (bypass antispam for this domain) $ crockery blacklist receipt (no emails to be permitted from this domain diff --git a/cmd/crockery/cmd_init.go b/cmd/crockery/cmd_init.go index a428242..955c7d5 100644 --- a/cmd/crockery/cmd_init.go +++ b/cmd/crockery/cmd_init.go @@ -9,19 +9,20 @@ import ( "gopkg.in/urfave/cli.v1" - "ur.gs/crockery/internal/imap" "ur.gs/crockery/internal/store" ) func crockeryInit(c *cli.Context) error { - db := c.GlobalString("db") + home := c.GlobalString("home") domain := c.String("domain") certFile := c.String("cert") keyFile := c.String("key") postmasterPassword := c.String("postmaster-password") - if db == "" { - return fmt.Errorf("No crockery database file specified") + db := store.BuildDatabasePath(home) + + if home == "" { + return fmt.Errorf("No crockery home directory specified") } if domain == "" { @@ -42,8 +43,13 @@ func crockeryInit(c *cli.Context) error { return fmt.Errorf("A password must be specified for the postmaster user") } + // TODO: check it is a directory + if _, err := os.Stat(home); os.IsNotExist(err) { + return fmt.Errorf("Home directory %q does not exist!", home) + } + if _, err := os.Stat(db); !os.IsNotExist(err) { - return fmt.Errorf("File %q already exists, refusing to overwrite", db) + return fmt.Errorf("Database file %q already exists!", db) } certPEM, err := ioutil.ReadFile(certFile) @@ -56,11 +62,11 @@ func crockeryInit(c *cli.Context) error { return fmt.Errorf("Failed to read key from %q: %v", keyFile, err) } - if err := store.Init(db, domain, certPEM, keyPEM); err != nil { - return fmt.Errorf("Couldn't initialize datase %q: %v", db, err) + if err := store.Init(home, domain, certPEM, keyPEM); err != nil { + return fmt.Errorf("Couldn't initialize home directory %q: %v", home, err) } - datastore, err := store.New(context.Background(), db) + datastore, err := store.New(context.Background(), home) if err != nil { return err } @@ -71,27 +77,27 @@ func crockeryInit(c *cli.Context) error { } username := "postmaster@" + domain - mailbox := store.Mailbox{ - Account: username, - Name: imap.InboxName, - } - - if err := datastore.CreateMailbox(&mailbox); err != nil { - return fmt.Errorf("Failed to create admin mailbox %s: %v", mailbox.Name, err) - } postmaster := store.Account{ - Username: username, - Admin: true, - PasswordHash: hash, - DefaultMailbox: mailbox.ID, // Automatically filled in by the store + Username: username, + Admin: true, + PasswordHash: hash, } if err := datastore.CreateAccount(&postmaster); err != nil { return fmt.Errorf("Failed to create admin account %s: %v", postmaster.Username, err) } - log.Printf("Created %q for domain %q. Next step:\n\t%s -db %s run", db, domain, os.Args[0], db) + maildir := store.Maildir{Account: postmaster, Name: ""} + + if err := datastore.CreateMaildir(&maildir); err != nil { + return fmt.Errorf("Failed to create admin maildir %q: %v", maildir.Rel(), err) + } + + log.Printf( + "Initialized %q for domain %q. Next step:\n\t%s -home %q run", + home, domain, os.Args[0], home, + ) return nil } diff --git a/cmd/crockery/cmd_run.go b/cmd/crockery/cmd_run.go index d924152..510ffb8 100644 --- a/cmd/crockery/cmd_run.go +++ b/cmd/crockery/cmd_run.go @@ -14,16 +14,16 @@ import ( ) func crockeryRun(c *cli.Context) error { - db := c.GlobalString("db") + home := c.GlobalString("home") ctx, cancel := context.WithCancel(context.Background()) - if db == "" { - return fmt.Errorf("No crockery database file specified") + if home == "" { + return fmt.Errorf("No crockery home directory specified") } - log.Printf("Loading config from file %q", db) + log.Printf("Loading config from directory %q", home) - datastore, err := store.New(context.Background(), db) + datastore, err := store.New(context.Background(), home) if err != nil { return err } @@ -31,7 +31,7 @@ func crockeryRun(c *cli.Context) error { defer datastore.Close() if datastore.Domain() == "" { - return fmt.Errorf("No domain configured in file %q", db) + return fmt.Errorf("No domain configured for Crockery home directory %q", home) } if _, err := datastore.FindAccount("postmaster@" + datastore.Domain()); err != nil { diff --git a/cmd/crockery/main.go b/cmd/crockery/main.go index bb80b59..6a92b1e 100644 --- a/cmd/crockery/main.go +++ b/cmd/crockery/main.go @@ -18,10 +18,10 @@ func main() { app.Flags = []cli.Flag{ cli.StringFlag{ - Name: "db", - Usage: "Database file to use", - EnvVar: "CROCKERY_DB", - Value: "crockery.db", + Name: "home", + Usage: "Home directory to use", + EnvVar: "HOME", + Value: os.Getenv("HOME"), }, } diff --git a/internal/imap/mailbox.go b/internal/imap/mailbox.go index b5c6c7f..0312094 100644 --- a/internal/imap/mailbox.go +++ b/internal/imap/mailbox.go @@ -1,6 +1,7 @@ package imap import ( + "fmt" "log" "time" @@ -14,23 +15,29 @@ var ( ) type Mailbox struct { - stored store.Mailbox + stored store.Maildir session *Session } // Name returns this mailbox name. func (m *Mailbox) Name() string { + if m.stored.Name == "" { + return InboxName + } + return m.stored.Name } // Info returns this mailbox info. func (m *Mailbox) Info() (*imap.MailboxInfo, error) { - log.Printf("Mailbox(%v).Info()", m.stored.ID) - return &imap.MailboxInfo{ + info := &imap.MailboxInfo{ Attributes: []string{}, // FIXME: what do we need here? Delimiter: ".", Name: m.Name(), - }, nil + } + + log.Printf("Mailbox(%v).Info(): %#v", m.stored.Rel(), info) + return info, nil } // Status returns this mailbox status. The fields Name, Flags, PermanentFlags @@ -38,7 +45,7 @@ func (m *Mailbox) Info() (*imap.MailboxInfo, error) { // This function does not affect the state of any messages in the mailbox. See // RFC 3501 section 6.3.10 for a list of items that can be requested. func (m *Mailbox) Status(items []string) (*imap.MailboxStatus, error) { - log.Printf("Mailbox(%v).Status(%#v)", m.stored.ID, items) + log.Printf("Mailbox(%v).Status(%#v)", m.stored.Rel(), items) return &imap.MailboxStatus{ Name: m.Name(), @@ -52,7 +59,7 @@ func (m *Mailbox) Status(items []string) (*imap.MailboxStatus, error) { // SetSubscribed adds or removes the mailbox to the server's set of "active" // or "subscribed" mailboxes. func (m *Mailbox) SetSubscribed(subscribed bool) error { - log.Printf("Mailbox(%v).SetSubscribed(%#v): not implemented", m.stored.ID, subscribed) + log.Printf("Mailbox(%v).SetSubscribed(%#v): not implemented", m.stored.Rel(), subscribed) return ErrNotImplemented } @@ -63,7 +70,7 @@ func (m *Mailbox) SetSubscribed(subscribed bool) error { // real time to complete. If a server implementation has no such housekeeping // considerations, CHECK is equivalent to NOOP. func (m *Mailbox) Check() error { - log.Printf("Mailbox(%v).Check()", m.stored.ID) + log.Printf("Mailbox(%v).Check()", m.stored.Rel()) return nil // FIXME: do we need to do anything here? } @@ -73,7 +80,7 @@ func (m *Mailbox) Check() error { // // Messages must be sent to ch. When the function returns, ch must be closed. func (m *Mailbox) ListMessages(uid bool, seqset *imap.SeqSet, items []string, ch chan<- *imap.Message) error { - log.Printf("Mailbox(%v).ListMessages(%#v, %#v, %#v, ch)", m.stored.ID, uid, *seqset, items) + log.Printf("Mailbox(%v).ListMessages(%#v, %#v, %#v, ch)", m.stored.Rel(), uid, *seqset, items) defer close(ch) var messages []store.Message @@ -92,27 +99,35 @@ func (m *Mailbox) ListMessages(uid bool, seqset *imap.SeqSet, items []string, ch log.Printf(" %v messages: %#v", len(messages), messages) for _, message := range messages { + header, err := message.Header() + if err != nil { + return err + } + envelope := &imap.Envelope{ // Date: - Subject: message.Header.Get("Subject"), + Subject: header.Get("Subject"), // From: // Sender: // ReplyTo: // To: // Cc: // Bcc: - InReplyTo: message.Header.Get("In-Reply-To"), - MessageId: message.Header.Get("Message-Id"), + InReplyTo: header.Get("In-Reply-To"), + MessageId: header.Get("Message-Id"), } body := map[*imap.BodySectionName]imap.Literal{} - imapMsg := imap.NewMessage(uint32(message.ID), items) - fillItems(imapMsg, message) + imapMsg := imap.NewMessage(uint32(message.UID), items) + if err := fillItems(imapMsg, message); err != nil { + return err + } + imapMsg.Envelope = envelope imapMsg.Body = body - imapMsg.Uid = uint32(message.ID) - imapMsg.Size = uint32(len(message.EML)) + imapMsg.Uid = uint32(message.UID) + imapMsg.Size = uint32(message.Len()) log.Printf(" Sending message %#v", imapMsg) ch <- imapMsg @@ -122,28 +137,34 @@ func (m *Mailbox) ListMessages(uid bool, seqset *imap.SeqSet, items []string, ch return nil } -func fillItems(imapMsg *imap.Message, msg store.Message) { +func fillItems(imapMsg *imap.Message, msg store.Message) error { for k, _ := range imapMsg.Items { switch k { case "UID": - imapMsg.Items[k] = msg.ID + imapMsg.Items[k] = msg.UID case "FLAGS": imapMsg.Items[k] = []string{} case "RFC822.SIZE": - imapMsg.Items[k] = uint32(len(msg.EML)) - // FIXME: Actually send *just* the header... + imapMsg.Items[k] = uint32(msg.Len()) case "RFC822.HEADER": - imapMsg.Items[k] = string(msg.EML) + str, err := msg.HeaderString() + if err != nil { + return err + } + + imapMsg.Items[k] = str default: - log.Printf("Unknown item: %v", k) + return fmt.Errorf("Unknown item: %v", k) } } + + return nil } // SearchMessages searches messages. The returned list must contain UIDs if // uid is set to true, or sequence numbers otherwise. func (m *Mailbox) SearchMessages(uid bool, criteria *imap.SearchCriteria) ([]uint32, error) { - log.Printf("Mailbox(%v).SearchMessages(): not implemented", m.stored.ID) + log.Printf("Mailbox(%v).SearchMessages(): not implemented", m.stored.Rel()) return nil, ErrNotImplemented } @@ -154,7 +175,7 @@ func (m *Mailbox) SearchMessages(uid bool, criteria *imap.SearchCriteria) ([]uin // If the Backend implements Updater, it must notify the client immediately // via a mailbox update. func (m *Mailbox) CreateMessage(flags []string, date time.Time, body imap.Literal) error { - log.Printf("Mailbox(%v).CreateMessage(%#v %#v %#v): not implemented", m.stored.ID, flags, date, body) + log.Printf("Mailbox(%v).CreateMessage(%#v %#v %#v): not implemented", m.stored.Rel(), flags, date, body) return ErrNotImplemented } @@ -163,7 +184,7 @@ func (m *Mailbox) CreateMessage(flags []string, date time.Time, body imap.Litera // If the Backend implements Updater, it must notify the client immediately // via a message update. func (m *Mailbox) UpdateMessagesFlags(uid bool, seqset *imap.SeqSet, operation imap.FlagsOp, flags []string) error { - log.Printf("Mailbox(%v).UpdateMessagesFlags(...): not implemented", m.stored.ID) + log.Printf("Mailbox(%v).UpdateMessagesFlags(...): not implemented", m.stored.Rel()) return ErrNotImplemented } @@ -177,7 +198,7 @@ func (m *Mailbox) UpdateMessagesFlags(uid bool, seqset *imap.SeqSet, operation i // If the Backend implements Updater, it must notify the client immediately // via a mailbox update. func (m *Mailbox) CopyMessages(uid bool, seqset *imap.SeqSet, dest string) error { - log.Printf("Mailbox(%v).CopyMessages(...): not implemented", m.stored.ID) + log.Printf("Mailbox(%v).CopyMessages(...): not implemented", m.stored.Rel()) return ErrNotImplemented } @@ -187,6 +208,6 @@ func (m *Mailbox) CopyMessages(uid bool, seqset *imap.SeqSet, dest string) error // If the Backend implements Updater, it must notify the client immediately // via an expunge update. func (m *Mailbox) Expunge() error { - log.Printf("Mailbox(%v).CopyMessages(): not implemented", m.stored.ID) + log.Printf("Mailbox(%v).CopyMessages(): not implemented", m.stored.Rel()) return ErrNotImplemented } diff --git a/internal/imap/session.go b/internal/imap/session.go index 1cffca4..9bc2670 100644 --- a/internal/imap/session.go +++ b/internal/imap/session.go @@ -27,7 +27,7 @@ func (s *Session) Username() string { } func (s *Session) ListMailboxes(subscribed bool) ([]imapbackend.Mailbox, error) { - mailboxes, err := s.store.FindMailboxes(s.Account) + mailboxes, err := s.store.FindMaildirs(s.Account) log.Printf("Session(%v).ListMailboxes(%v): %#v %#v", s.ID, subscribed, mailboxes, err) if err != nil { @@ -43,13 +43,16 @@ func (s *Session) ListMailboxes(subscribed bool) ([]imapbackend.Mailbox, error) } func (s *Session) GetMailbox(name string) (imapbackend.Mailbox, error) { - mailbox, err := s.store.FindMailbox(s.Account, name) - log.Printf("Session(%v).GetMailbox(%v): %#v %#v", s.ID, name, mailbox, err) + if name == InboxName { + name = "" + } + maildir, err := s.store.FindMaildir(s.Account, name) + log.Printf("Session(%v).GetMailbox(%v): %#v %#v", s.ID, name, maildir, err) if err != nil { return nil, err } - return &Mailbox{stored: mailbox, session: s}, nil + return &Mailbox{stored: maildir, session: s}, nil } func (s *Session) CreateMailbox(name string) error { diff --git a/internal/imap/uidlist/entry.go b/internal/imap/uidlist/entry.go new file mode 100644 index 0000000..682e005 --- /dev/null +++ b/internal/imap/uidlist/entry.go @@ -0,0 +1,65 @@ +package uidlist + +import ( + "errors" + "fmt" + "strings" +) + +type Entry struct { + UID uint64 + Extra map[string][]string + LastKnownFilename string +} + +func ParseEntry(line string) (Entry, error) { + var out Entry + + parts := strings.SplitN(strings.TrimSpace(line), ":", 2) + if len(parts) != 2 { + return out, errors.New("Couldn't find start of filename") + } + + attrs := strings.Split(parts[0], " ") + if len(attrs) == 0 { + return out, errors.New("Couldn't find UID") + } + + if err := setUint64(attrs[0], &out.UID); err != nil { + return out, err + } + + out.Extra = make(map[string][]string) + // FIXME: this is all disturbingly similar to header.go + for _, attr := range attrs[1:] { + if len(attr) == 0 { + continue + } + + key := attr[0:1] + value := attr[1:] + out.Extra[key] = append(out.Extra[key], value) + } + + out.LastKnownFilename = parts[1] + + return out, nil +} + +func (e *Entry) Key() string { + parts := strings.SplitN(e.LastKnownFilename, ":", 2) + + return parts[0] +} + +func (e *Entry) String() string { + // FIXME: also disturbingly similar to header.go + var sb strings.Builder + fmt.Fprintf(&sb, "%d", e.UID) + for k, v := range e.Extra { + fmt.Fprintf(&sb, " %s%s", k, v) + } + fmt.Fprintf(&sb, " :%s", e.LastKnownFilename) + + return sb.String() +} diff --git a/internal/imap/uidlist/header.go b/internal/imap/uidlist/header.go new file mode 100644 index 0000000..1626b76 --- /dev/null +++ b/internal/imap/uidlist/header.go @@ -0,0 +1,131 @@ +package uidlist + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" + "time" +) + +const ( + HeaderTemplate = "%d V%d N%d G%s" +) + +type Header struct { + Version uint64 + UIDValidity uint64 + NextUID uint64 + GUID string + Extra map[string][]string +} + +func setUint64(s string, out *uint64) error { + num, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return err + } + + *out = uint64(num) + return nil +} + +// GenerateHeader produces a valid verison 3 dovecot-uidlist header line, using +// the time for UIDValidity and a random string for the GUID. +func GenerateHeader() (Header, error) { + // A GUID is a random 128-bit number + guidBytes := make([]byte, 16) + _, err := rand.Read(guidBytes) + if err != nil { + return Header{}, err + } + + // ...hex-encoded + guidString := hex.EncodeToString(guidBytes) + + return Header{ + Version: uint64(3), + UIDValidity: uint64(time.Now().Unix()), + NextUID: uint64(1), + GUID: guidString, + Extra: make(map[string][]string), + }, nil +} + +// ParseHeader breaks a dovecot-uidlist header line into its constituent parts +func ParseHeader(line string) (Header, error) { + var seenUIDValidity bool + var seenNextUID bool + var seenGUID bool + + out := Header{} + parts := strings.Split(strings.TrimSpace(line), " ") + if len(parts) < 4 { + return out, errors.New("Short header") + } + + if err := setUint64(parts[0], &out.Version); err != nil { + return out, err + } + + out.Extra = make(map[string][]string) + for _, part := range parts[1:] { + part = strings.TrimSpace(part) + + if len(part) == 0 { + continue + } + + key := part[0:1] + value := part[1:] + switch key { + case "V": + if seenUIDValidity { + return out, errors.New("UIDVALIDITY specified twice") + } + seenUIDValidity = true + + if err := setUint64(value, &out.UIDValidity); err != nil { + return out, err + } + case "N": + if seenNextUID { + return out, errors.New("NEXTUID specified twice") + } + seenNextUID = true + + if err := setUint64(value, &out.NextUID); err != nil { + return out, err + } + case "G": + if seenGUID { + return out, errors.New("GUID specified twice") + } + seenGUID = true + + out.GUID = value + default: + out.Extra[key] = append(out.Extra[key], value) + } + } + + return out, nil +} + +func (h Header) String() string { + var sb strings.Builder + + fmt.Fprintf( + &sb, + HeaderTemplate, + h.Version, h.UIDValidity, h.NextUID, h.GUID, + ) + + for k, v := range h.Extra { + fmt.Fprintf(&sb, " %s%s", k, v) + } + + return sb.String() +} diff --git a/internal/imap/uidlist/header_test.go b/internal/imap/uidlist/header_test.go new file mode 100644 index 0000000..582e3ee --- /dev/null +++ b/internal/imap/uidlist/header_test.go @@ -0,0 +1,30 @@ +package uidlist_test + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "ur.gs/crockery/internal/imap/uidlist" +) + +func TestParseHeaderValid(t *testing.T) { + valid := " 3 V1275660208 Afortytwo N25022 G3085f01b7f11094c501100008c4a11c1 \r\n" + + hdr, err := uidlist.ParseHeader(valid) + require.NoError(t, err) + + assert.Equal(t, uint64(3), hdr.Version) + assert.Equal(t, uint64(1275660208), hdr.UIDValidity) + assert.Equal(t, uint64(25022), hdr.NextUID) + assert.Equal(t, "3085f01b7f11094c501100008c4a11c1", hdr.GUID) + assert.Equal(t, map[string][]string{"A": []string{"fortytwo"}}, hdr.Extra) +} + +func TestParseHeaderEmpty(t *testing.T) { + hdr, err := uidlist.ParseHeader("") + require.Error(t, err, errors.New("Short header")) + assert.Equal(t, uidlist.Header{}, hdr) +} diff --git a/internal/imap/uidlist/uidlist.go b/internal/imap/uidlist/uidlist.go new file mode 100644 index 0000000..9f3a36a --- /dev/null +++ b/internal/imap/uidlist/uidlist.go @@ -0,0 +1,233 @@ +// package uidlist implements Dovecot's `dovecot-uidlist` file format for +// Maildir directories +// +// Reference: https://wiki.dovecot.org/MailboxFormat/Maildir +package uidlist + +import ( + "bufio" + "fmt" + "log" + "os" + "path/filepath" + "sync" +) + +const ( + DefaultFilename = "dovecot-uidlist" +) + +type List struct { + filename string + header Header + entries []Entry + + mutex *sync.Mutex +} + +func Create(dir string) (*List, error) { + return CreateFile(filepath.Join(dir, DefaultFilename)) +} + +// FIXME: on startup, we need to scan `cur/` in case any problems arose +func Open(dir string) (*List, error) { + return OpenFile(filepath.Join(dir, DefaultFilename)) +} + +func CreateFile(filename string) (*List, error) { + list, err := handleFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC) + if err != nil { + return nil, err + } + + hdr, err := GenerateHeader() + if err != nil { + return nil, err + } + + list.header = hdr + + // TODO: allow the dovecot-uidlist to be populated from scratch + // FIXME: in particular, NextUID and entries are wrong here + + if err := list.save(); err != nil { + return nil, err + } + + return list, nil +} + +func OpenFile(filename string) (*List, error) { + list, err := handleFile(filename, os.O_RDWR|os.O_CREATE) + if err != nil { + return nil, err + } + + if err := list.parse(); err != nil { + return nil, err + } + + return list, nil +} + +func handleFile(filename string, flags int) (*List, error) { + file, err := os.OpenFile(filename, flags, 0600) + if err != nil { + return nil, err + } + file.Close() + + return &List{ + filename: filename, + mutex: &sync.Mutex{}, + }, nil +} + +func (l *List) UIDValidity() uint64 { + return l.header.UIDValidity +} + +func (l *List) GUID() string { + return l.header.GUID +} + +func (l *List) Entries() []Entry { + l.mutex.Lock() + defer l.mutex.Unlock() + + // FIXME: use copy() ? + return l.entries +} + +func (l *List) EntriesInRange(start, end uint64) []Entry { + l.mutex.Lock() + defer l.mutex.Unlock() + + // FIXME: this shouldn't need a full scan + var out []Entry + + log.Printf("Checking range %d to %d in %#v", start, end, l.entries) + + for _, e := range l.entries { + if e.UID >= start && (end == 0 || e.UID <= end) { + out = append(out, e) + } + } + + return out +} + +func (l *List) Entry(key string) *Entry { + l.mutex.Lock() + defer l.mutex.Unlock() + + return nil +} + +func (l *List) Append(filenames ...string) error { + l.mutex.Lock() + defer l.mutex.Unlock() + + file, err := os.OpenFile(l.filename, os.O_APPEND|os.O_RDWR, 0600) + if err != nil { + return err + } + + defer file.Close() + + for _, filename := range filenames { + entry := Entry{ + UID: l.header.NextUID, + LastKnownFilename: filename, + } + log.Printf("Adding entry: %#v", entry) + + l.entries = append(l.entries, entry) + l.header.NextUID = l.header.NextUID + 1 + + // Try to write them all, at least + // FIXME: we don't bubble up errors when we should + if _, err := fmt.Fprintln(file, entry.String()); err != nil { + log.Printf("failed to write UID %d to %s: %v", entry.UID, l.filename, err) + } + } + log.Printf("Appended %d entries", len(filenames)) + + return nil +} + +func (l *List) save() (err error) { + l.mutex.Lock() + defer l.mutex.Unlock() + + // FIXME: When rewriting the file, we're meant to: + // * Take the dovecot-uidlist.lock file + // * write to a temporary file then atomically rename onto -uidlist + // Unsure if the temporary file and the lock file are the same thing or not + + file, err := os.OpenFile(l.filename, os.O_TRUNC|os.O_RDWR, 0600) + if err != nil { + return + } + + defer func() { err = file.Close() }() + + _, err = fmt.Fprintln(file, l.header.String()) + if err != nil { + return + } + + for _, e := range l.entries { + _, err = fmt.Fprintln(file, e.String()) + if err != nil { + return + } + } + + return +} + +func (l *List) parse() error { + l.mutex.Lock() + defer l.mutex.Unlock() + + file, err := os.Open(l.filename) + if err != nil { + return err + } + + defer file.Close() + + r := bufio.NewScanner(file) + + // Get the header if present, which is the first line + if !r.Scan() { + return r.Err() + } + + header, err := ParseHeader(r.Text()) + if err != nil { + return err + } + + // Now parse each additional line + entries := []Entry{} + for r.Scan() { + entry, err := ParseEntry(r.Text()) + if err != nil { + return err + } + entries = append(entries, entry) + + // If NextUID is out of date in the file, ignore it. We never want to + // re-use a seen UID + if entry.UID >= l.header.NextUID { + l.header.NextUID++ + } + } + + l.header = header + l.entries = entries + + return nil +} diff --git a/internal/smtp/msa.go b/internal/smtp/msa.go index 2ae0ae5..23da24e 100644 --- a/internal/smtp/msa.go +++ b/internal/smtp/msa.go @@ -66,12 +66,12 @@ func (m *msa) Login(user, pass string) (smtp.User, error) { Handler: &sender{msa: m, account: account}, } - log.Printf("Beginning session %d for username=%s", session.ID, account.Username) + log.Printf("Beginning session %s for username=%s", session.ID, account.Username) // FIXME: TODO: Track ongoing sessions for termination or notifications return session, nil } func (m *msa) AnonymousLogin() (smtp.User, error) { - return nil, smtp.AuthRequiredErr + return nil, smtp.ErrAuthRequired } diff --git a/internal/smtp/session.go b/internal/smtp/session.go index 5a71949..eecdabf 100644 --- a/internal/smtp/session.go +++ b/internal/smtp/session.go @@ -15,14 +15,14 @@ type Session struct { } func (s *Session) Send(from string, to []string, r io.Reader) error { - log.Printf("session=%d from=%s to=%s", s.ID, from, to) + log.Printf("session=%s from=%s to=%#v", s.ID, from, to) // TODO: a chain of middlewares here return s.Handler.ServeSMTP(from, to, r) } func (s *Session) Logout() error { - log.Printf("Ending session %d", s.ID) + log.Printf("Ending session %s", s.ID) return nil } diff --git a/internal/store/account.go b/internal/store/account.go index a46d2c2..9edf7bb 100644 --- a/internal/store/account.go +++ b/internal/store/account.go @@ -29,9 +29,6 @@ type Account struct { // As generated by HashPassword PasswordHash string - - // Where to put messages by default. FK: mailbox.id - DefaultMailbox uint64 } // HashPassword turns a plaintext password into a crypt()ed string, using bcrypt diff --git a/internal/store/maildir.go b/internal/store/maildir.go index 317a76f..31873f7 100644 --- a/internal/store/maildir.go +++ b/internal/store/maildir.go @@ -2,42 +2,123 @@ package store import ( "errors" + "net/mail" + "os" + "path/filepath" + + "github.com/luksen/maildir" + + "ur.gs/crockery/internal/imap/uidlist" ) -type MailboxInterface interface { - CreateMailbox(*Mailbox) error - FindMailbox(account Account, name string) (Mailbox, error) - FindMailboxes(account Account) ([]Mailbox, error) +type MaildirInterface interface { + CreateMaildir(*Maildir) error + FindMaildir(account Account, name string) (Maildir, error) + FindMaildirs(account Account) ([]Maildir, error) + FindMessagesInRange(m Maildir, start, end uint64) ([]Message, error) } -type Mailbox struct { - ID uint64 `storm:"id,increment"` - Account string `storm:"index"` // FK accounts.username - Name string `storm:"index"` +type Maildir struct { + Account Account + Name string + + directory string + + filesystem maildir.Dir + uids *uidlist.List } -func (c *concrete) CreateMailbox(mailbox *Mailbox) error { - return c.storm.Save(mailbox) +func (m *Maildir) Rel() string { + return filepath.Join(m.Account.Username, m.Name) } -func (c *concrete) FindMailbox(account Account, name string) (Mailbox, error) { - // FIXME I don't know how to storm - candidates, err := c.FindMailboxes(account) +func (m *Maildir) Directory() string { + return m.directory +} + +func (m *Maildir) Header(key string) (mail.Header, error) { + return m.filesystem.Header(key) +} + +func (c *concrete) CreateMaildir(m *Maildir) error { + if m.directory == "" { + m.directory = c.buildMaildirPath(m.Account, m.Name) + } + + if string(m.filesystem) == "" { + m.filesystem = maildir.Dir(m.directory) + } + + err := m.filesystem.Create() if err != nil { - return Mailbox{}, err + return err } - for _, mbox := range candidates { - if mbox.Name == name { - return mbox, nil + if m.uids == nil { + uids, err := uidlist.Create(m.directory) + if err != nil { + // FIXME: this leaves a dangling Maildir + return err } + + m.uids = uids } - return Mailbox{}, errors.New("not found") + return nil } -func (c *concrete) FindMailboxes(account Account) ([]Mailbox, error) { - var out []Mailbox +func (c *concrete) FindMaildir(account Account, name string) (Maildir, error) { + // FIXME: multiple IMAP sessions get different Maildir instances, so breaking + // any hope at locking (sigh) + m := Maildir{ + Account: account, + Name: name, + directory: c.buildMaildirPath(account, name), + } + m.filesystem = maildir.Dir(m.directory) - return out, c.storm.Find("Account", account.Username, &out) + if _, err := os.Stat(m.Directory()); os.IsNotExist(err) { + return Maildir{}, errors.New("not found") + } + + uids, err := uidlist.Open(m.directory) + if err != nil { + return Maildir{}, err + } + m.uids = uids + + return m, nil +} + +func (c *concrete) FindMaildirs(account Account) ([]Maildir, error) { + // TODO: iterate through the maildir looking for submaildirs + defaultMaildir, err := c.FindMaildir(account, "") + if err != nil { + return nil, err + } + + return []Maildir{defaultMaildir}, nil +} + +func (c *concrete) buildMaildirPath(account Account, parts ...string) string { + bits := append([]string{BuildMailboxesPath(c.home), account.Username}, parts...) + + return filepath.Join(bits...) +} + +func (c *concrete) FindMessagesInRange(m Maildir, start, end uint64) ([]Message, error) { + var out []Message + entries := m.uids.EntriesInRange(start, end) + + for _, entry := range entries { + msg := Message{ + UID: entry.UID, + Maildir: m, + Key: entry.Key(), + } + + out = append(out, msg) + } + + return out, nil } diff --git a/internal/store/message.go b/internal/store/message.go index aae03a4..152083a 100644 --- a/internal/store/message.go +++ b/internal/store/message.go @@ -1,50 +1,108 @@ package store import ( + "fmt" + "io" + "log" "net/mail" - - "github.com/asdine/storm/q" + "path/filepath" + "strings" ) +type Message struct { + Maildir Maildir // maildir.name + + UID uint64 + Key string + Data io.ReadCloser + hdrCache mail.Header +} + +// TODO: the cache is probably fine +func (m *Message) Header() (mail.Header, error) { + if m.hdrCache != nil { + return m.hdrCache, nil + } + + hdr, err := m.Maildir.Header(m.Key) + if err != nil { + return nil, err + } + + m.hdrCache = hdr + return hdr, nil +} + +func (m *Message) Len() uint64 { + return 0 // FIXME +} + +func (m *Message) HeaderString() (string, error) { + var sb strings.Builder + + hdr, err := m.Header() + if err != nil { + return "", err + } + + for k, vs := range hdr { + for _, v := range vs { + fmt.Fprintf(&sb, "%s: %s\r\n", k, v) + } + } + return sb.String(), nil // FIXME +} + type MessageInterface interface { CreateMessage(Message) error - FindMessages(Mailbox) ([]Message, error) - - // if end == 0, it means "from start to most recent" - FindMessagesInRange(mailbox Mailbox, start, end uint64) ([]Message, error) -} - -type Message struct { - // FIXME: this should be per-mailbox, IMAP "only" allows 32-bit message UIDs? - ID uint64 `storm:"id,increment"` - - Mailbox uint64 `storm:"index"` // FK mailbox.id - - Header mail.Header // The parsed headers - EML []byte // The full email, unaltered } +// DeliverMessage takes the given message and delivers it to the correct maildir +// It does not call Close() on message.Data func (c *concrete) CreateMessage(message Message) error { - return c.storm.Save(&message) -} + // TODO: recover from panics, assure Abort() is called for them + maildir := message.Maildir -func (c *concrete) FindMessages(mailbox Mailbox) ([]Message, error) { - var out []Message - - return out, c.storm.Find("Mailbox", mailbox.ID, &out) -} - -func (c *concrete) FindMessagesInRange(mailbox Mailbox, start, end uint64) ([]Message, error) { - var out []Message - - matchers := []q.Matcher{ - q.Eq("Mailbox", mailbox.ID), - q.Gte("ID", start), + delivery, err := maildir.filesystem.NewDelivery() + if err != nil { + return err } - if end > 0 { - matchers = append(matchers, q.Lte("ID", end)) + if _, err := io.Copy(delivery, message.Data); err != nil { + delivery.Abort() + return err } - return out, c.storm.Select(matchers...).Find(&out) + if err := delivery.Close(); err != nil { + delivery.Abort() + return err + } + + // FIXME: this moves files in the maildir before the dovecot-uidlist is + // updated. Not ideal. + // FIXME: we're not + keys, err := maildir.filesystem.Unseen() + if err != nil { + log.Printf("Failed moving messages from new/ to cur/: %v", err) + } + + var filenames []string + for _, key := range keys { + filename, err := maildir.filesystem.Filename(key) + if err != nil { + log.Printf("Failed getting filename for unseen message %q: %v", key, err) + continue + } + filenames = append(filenames, filepath.Base(filename)) + } + + // Don't bubble the error up here as delivery has succeeded, IMAP just + // doesn't know it has. We don't want the client to attempt redelivery. + // + // FIXME: this is IMAP stuff that needs moving + if err := maildir.uids.Append(filenames...); err != nil { + log.Printf("Allocating UIDs for unseen messages failed: %v", err) + } + + return nil } diff --git a/internal/store/spool.go b/internal/store/spool.go index 7aa7990..c89b568 100644 --- a/internal/store/spool.go +++ b/internal/store/spool.go @@ -5,40 +5,47 @@ import ( "io" "io/ioutil" "log" - "net/mail" + // "net/mail" + + "golang.org/x/sync/errgroup" ) type SpoolInterface interface { SpoolMessage([]Account, io.Reader) error } -// FIXME: we don't actually spool for now, we just store it to each user's -// mailbox. This might lead to oddness in the partial delivery case. +func (c *concrete) deliver(account Account, data []byte) error { + // FIXME: this could be a FindOrCreateMaildir call for atomic create? + maildir, err := c.FindMaildir(account, "") + if err != nil { + return err + } + + message := Message{ + Maildir: maildir, + Data: ioutil.NopCloser(bytes.NewReader(data)), + } + + log.Printf("Creating email for %#v", account) + log.Print(string(data)) + + return c.CreateMessage(message) +} + +// FIXME: we don't actually spool for now, we just store it in each user's +// maildir. This might lead to oddness in the partial delivery case. func (c *concrete) SpoolMessage(recipients []Account, r io.Reader) error { data, err := ioutil.ReadAll(r) if err != nil { return err } - message, err := mail.ReadMessage(bytes.NewReader(data)) - if err != nil { - return err - } + var g errgroup.Group for _, recipient := range recipients { - message := Message{ - Mailbox: recipient.DefaultMailbox, - Header: message.Header, - EML: data, - } - - log.Printf("Creating email: %#v", message) - log.Print(string(data)) - - if err := c.CreateMessage(message); err != nil { - return err - } + recipient := recipient + g.Go(func() error { return c.deliver(recipient, data) }) } - return nil + return g.Wait() } diff --git a/internal/store/store.go b/internal/store/store.go index 83527b6..dc285e1 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -5,11 +5,18 @@ import ( "crypto/tls" "fmt" "io" + "os" + "path/filepath" "github.com/asdine/storm" "github.com/coreos/bbolt" ) +const ( + DatabaseFilename = "crockery.db" + MailboxesDirname = "mailboxes" +) + type Interface interface { Domain() string TLS() tls.Certificate @@ -19,26 +26,42 @@ type Interface interface { SetTLS([]byte, []byte) error AccountInterface - MailboxInterface + MaildirInterface MessageInterface SpoolInterface io.Closer } +func buildPath(parts ...string) string { + if len(parts) == 0 || parts[0] == "" { + return "" + } + + return filepath.Join(parts...) +} +func BuildDatabasePath(home string) string { + return buildPath(home, DatabaseFilename) +} + +func BuildMailboxesPath(home string) string { + return buildPath(home, MailboxesDirname) +} + func IsNotFound(err error) bool { return err.Error() == "not found" // Magic hardcoded value in storm } -func New(ctx context.Context, filename string) (Interface, error) { +func New(ctx context.Context, home string) (Interface, error) { + filename := BuildDatabasePath(home) db, err := storm.Open(filename, storm.BoltOptions(0600, &bolt.Options{})) if err != nil { return nil, err } out := &concrete{ - filename: filename, - storm: db, + home: home, + storm: db, } if err := out.setup(); err != nil { @@ -48,19 +71,30 @@ func New(ctx context.Context, filename string) (Interface, error) { return out, nil } -func Init(filename, domain string, certPEM, keyPEM []byte) error { +func Init(home, domain string, certPEM, keyPEM []byte) error { + filename := BuildDatabasePath(home) + mailboxes := BuildMailboxesPath(home) + + if _, err := os.Stat(mailboxes); !os.IsNotExist(err) { + return fmt.Errorf("mailboxes directory %q already exists", mailboxes) + } + db, err := storm.Open(filename, storm.BoltOptions(0600, &bolt.Options{})) if err != nil { return err } builder := &concrete{ - filename: filename, - storm: db, + home: home, + storm: db, } defer builder.Close() + if err := os.Mkdir(mailboxes, 0700); err != nil { + return fmt.Errorf("Couldn't create mailboxes directory %q: %v", mailboxes, err) + } + if err := builder.SetDomain(domain); err != nil { return fmt.Errorf("Couldn't set domain: %v", err) } @@ -73,8 +107,9 @@ func Init(filename, domain string, certPEM, keyPEM []byte) error { } type concrete struct { - filename string - storm *storm.DB + home string + storm *storm.DB + domainBucket storm.Node // These are persisted in BoltDB, but we might as well keep them in memory