lupine/crockery
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Convert from BoltDB to Maildir storage for emails

Browse Source
This commit is contained in:
Nick Thomas
2018-06-26 03:08:51 +01:00
parent d25ed6c1bd
commit a3c2508160
19 changed files with 869 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
internal/imap/mailbox.go
View File

@@ -1,6 +1,7 @@
package imap
import (
"fmt"
"log"
"time"
@@ -14,23 +15,29 @@ var (
)
type Mailbox struct {
stored store.Mailbox
stored store.Maildir
session *Session
}
// Name returns this mailbox name.
func (m *Mailbox) Name() string {
if m.stored.Name == "" {
return InboxName
}
return m.stored.Name
}
// Info returns this mailbox info.
func (m *Mailbox) Info() (*imap.MailboxInfo, error) {
log.Printf("Mailbox(%v).Info()", m.stored.ID)
return &imap.MailboxInfo{
info := &imap.MailboxInfo{
Attributes: []string{}, // FIXME: what do we need here?
Delimiter: ".",
Name: m.Name(),
}, nil
}
log.Printf("Mailbox(%v).Info(): %#v", m.stored.Rel(), info)
return info, nil
}
// Status returns this mailbox status. The fields Name, Flags, PermanentFlags
@@ -38,7 +45,7 @@ func (m *Mailbox) Info() (*imap.MailboxInfo, error) {
// This function does not affect the state of any messages in the mailbox. See
// RFC 3501 section 6.3.10 for a list of items that can be requested.
func (m *Mailbox) Status(items []string) (*imap.MailboxStatus, error) {
log.Printf("Mailbox(%v).Status(%#v)", m.stored.ID, items)
log.Printf("Mailbox(%v).Status(%#v)", m.stored.Rel(), items)
return &imap.MailboxStatus{
Name: m.Name(),
@@ -52,7 +59,7 @@ func (m *Mailbox) Status(items []string) (*imap.MailboxStatus, error) {
// SetSubscribed adds or removes the mailbox to the server's set of "active"
// or "subscribed" mailboxes.
func (m *Mailbox) SetSubscribed(subscribed bool) error {
log.Printf("Mailbox(%v).SetSubscribed(%#v): not implemented", m.stored.ID, subscribed)
log.Printf("Mailbox(%v).SetSubscribed(%#v): not implemented", m.stored.Rel(), subscribed)
return ErrNotImplemented
}
@@ -63,7 +70,7 @@ func (m *Mailbox) SetSubscribed(subscribed bool) error {
// real time to complete. If a server implementation has no such housekeeping
// considerations, CHECK is equivalent to NOOP.
func (m *Mailbox) Check() error {
log.Printf("Mailbox(%v).Check()", m.stored.ID)
log.Printf("Mailbox(%v).Check()", m.stored.Rel())
return nil // FIXME: do we need to do anything here?
}
@@ -73,7 +80,7 @@ func (m *Mailbox) Check() error {
//
// Messages must be sent to ch. When the function returns, ch must be closed.
func (m *Mailbox) ListMessages(uid bool, seqset *imap.SeqSet, items []string, ch chan<- *imap.Message) error {
log.Printf("Mailbox(%v).ListMessages(%#v, %#v, %#v, ch)", m.stored.ID, uid, *seqset, items)
log.Printf("Mailbox(%v).ListMessages(%#v, %#v, %#v, ch)", m.stored.Rel(), uid, *seqset, items)
defer close(ch)
var messages []store.Message
@@ -92,27 +99,35 @@ func (m *Mailbox) ListMessages(uid bool, seqset *imap.SeqSet, items []string, ch
log.Printf(" %v messages: %#v", len(messages), messages)
for _, message := range messages {
header, err := message.Header()
if err != nil {
return err
}
envelope := &imap.Envelope{
// Date:
Subject: message.Header.Get("Subject"),
Subject: header.Get("Subject"),
// From:
// Sender:
// ReplyTo:
// To:
// Cc:
// Bcc:
InReplyTo: message.Header.Get("In-Reply-To"),
MessageId: message.Header.Get("Message-Id"),
InReplyTo: header.Get("In-Reply-To"),
MessageId: header.Get("Message-Id"),
}
body := map[*imap.BodySectionName]imap.Literal{}
imapMsg := imap.NewMessage(uint32(message.ID), items)
fillItems(imapMsg, message)
imapMsg := imap.NewMessage(uint32(message.UID), items)
if err := fillItems(imapMsg, message); err != nil {
return err
}
imapMsg.Envelope = envelope
imapMsg.Body = body
imapMsg.Uid = uint32(message.ID)
imapMsg.Size = uint32(len(message.EML))
imapMsg.Uid = uint32(message.UID)
imapMsg.Size = uint32(message.Len())
log.Printf(" Sending message %#v", imapMsg)
ch <- imapMsg
@@ -122,28 +137,34 @@ func (m *Mailbox) ListMessages(uid bool, seqset *imap.SeqSet, items []string, ch
return nil
}
func fillItems(imapMsg *imap.Message, msg store.Message) {
func fillItems(imapMsg *imap.Message, msg store.Message) error {
for k, _ := range imapMsg.Items {
switch k {
case "UID":
imapMsg.Items[k] = msg.ID
imapMsg.Items[k] = msg.UID
case "FLAGS":
imapMsg.Items[k] = []string{}
case "RFC822.SIZE":
imapMsg.Items[k] = uint32(len(msg.EML))
// FIXME: Actually send *just* the header...
imapMsg.Items[k] = uint32(msg.Len())
case "RFC822.HEADER":
imapMsg.Items[k] = string(msg.EML)
str, err := msg.HeaderString()
if err != nil {
return err
}
imapMsg.Items[k] = str
default:
log.Printf("Unknown item: %v", k)
return fmt.Errorf("Unknown item: %v", k)
}
}
return nil
}
// SearchMessages searches messages. The returned list must contain UIDs if
// uid is set to true, or sequence numbers otherwise.
func (m *Mailbox) SearchMessages(uid bool, criteria *imap.SearchCriteria) ([]uint32, error) {
log.Printf("Mailbox(%v).SearchMessages(): not implemented", m.stored.ID)
log.Printf("Mailbox(%v).SearchMessages(): not implemented", m.stored.Rel())
return nil, ErrNotImplemented
}
@@ -154,7 +175,7 @@ func (m *Mailbox) SearchMessages(uid bool, criteria *imap.SearchCriteria) ([]uin
// If the Backend implements Updater, it must notify the client immediately
// via a mailbox update.
func (m *Mailbox) CreateMessage(flags []string, date time.Time, body imap.Literal) error {
log.Printf("Mailbox(%v).CreateMessage(%#v %#v %#v): not implemented", m.stored.ID, flags, date, body)
log.Printf("Mailbox(%v).CreateMessage(%#v %#v %#v): not implemented", m.stored.Rel(), flags, date, body)
return ErrNotImplemented
}
@@ -163,7 +184,7 @@ func (m *Mailbox) CreateMessage(flags []string, date time.Time, body imap.Litera
// If the Backend implements Updater, it must notify the client immediately
// via a message update.
func (m *Mailbox) UpdateMessagesFlags(uid bool, seqset *imap.SeqSet, operation imap.FlagsOp, flags []string) error {
log.Printf("Mailbox(%v).UpdateMessagesFlags(...): not implemented", m.stored.ID)
log.Printf("Mailbox(%v).UpdateMessagesFlags(...): not implemented", m.stored.Rel())
return ErrNotImplemented
}
@@ -177,7 +198,7 @@ func (m *Mailbox) UpdateMessagesFlags(uid bool, seqset *imap.SeqSet, operation i
// If the Backend implements Updater, it must notify the client immediately
// via a mailbox update.
func (m *Mailbox) CopyMessages(uid bool, seqset *imap.SeqSet, dest string) error {
log.Printf("Mailbox(%v).CopyMessages(...): not implemented", m.stored.ID)
log.Printf("Mailbox(%v).CopyMessages(...): not implemented", m.stored.Rel())
return ErrNotImplemented
}
@@ -187,6 +208,6 @@ func (m *Mailbox) CopyMessages(uid bool, seqset *imap.SeqSet, dest string) error
// If the Backend implements Updater, it must notify the client immediately
// via an expunge update.
func (m *Mailbox) Expunge() error {
log.Printf("Mailbox(%v).CopyMessages(): not implemented", m.stored.ID)
log.Printf("Mailbox(%v).CopyMessages(): not implemented", m.stored.Rel())
return ErrNotImplemented
}

11
internal/imap/session.go
View File

@@ -27,7 +27,7 @@ func (s *Session) Username() string {
}
func (s *Session) ListMailboxes(subscribed bool) ([]imapbackend.Mailbox, error) {
mailboxes, err := s.store.FindMailboxes(s.Account)
mailboxes, err := s.store.FindMaildirs(s.Account)
log.Printf("Session(%v).ListMailboxes(%v): %#v %#v", s.ID, subscribed, mailboxes, err)
if err != nil {
@@ -43,13 +43,16 @@ func (s *Session) ListMailboxes(subscribed bool) ([]imapbackend.Mailbox, error)
}
func (s *Session) GetMailbox(name string) (imapbackend.Mailbox, error) {
mailbox, err := s.store.FindMailbox(s.Account, name)
log.Printf("Session(%v).GetMailbox(%v): %#v %#v", s.ID, name, mailbox, err)
if name == InboxName {
name = ""
}
maildir, err := s.store.FindMaildir(s.Account, name)
log.Printf("Session(%v).GetMailbox(%v): %#v %#v", s.ID, name, maildir, err)
if err != nil {
return nil, err
}
return &Mailbox{stored: mailbox, session: s}, nil
return &Mailbox{stored: maildir, session: s}, nil
}
func (s *Session) CreateMailbox(name string) error {

65
internal/imap/uidlist/entry.go Normal file
View File

@@ -0,0 +1,65 @@
package uidlist
import (
"errors"
"fmt"
"strings"
)
type Entry struct {
UID uint64
Extra map[string][]string
LastKnownFilename string
}
func ParseEntry(line string) (Entry, error) {
var out Entry
parts := strings.SplitN(strings.TrimSpace(line), ":", 2)
if len(parts) != 2 {
return out, errors.New("Couldn't find start of filename")
}
attrs := strings.Split(parts[0], " ")
if len(attrs) == 0 {
return out, errors.New("Couldn't find UID")
}
if err := setUint64(attrs[0], &out.UID); err != nil {
return out, err
}
out.Extra = make(map[string][]string)
// FIXME: this is all disturbingly similar to header.go
for _, attr := range attrs[1:] {
if len(attr) == 0 {
continue
}
key := attr[0:1]
value := attr[1:]
out.Extra[key] = append(out.Extra[key], value)
}
out.LastKnownFilename = parts[1]
return out, nil
}
func (e *Entry) Key() string {
parts := strings.SplitN(e.LastKnownFilename, ":", 2)
return parts[0]
}
func (e *Entry) String() string {
// FIXME: also disturbingly similar to header.go
var sb strings.Builder
fmt.Fprintf(&sb, "%d", e.UID)
for k, v := range e.Extra {
fmt.Fprintf(&sb, " %s%s", k, v)
}
fmt.Fprintf(&sb, " :%s", e.LastKnownFilename)
return sb.String()
}

131
internal/imap/uidlist/header.go Normal file
View File

@@ -0,0 +1,131 @@
package uidlist
import (
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"strconv"
"strings"
"time"
)
const (
HeaderTemplate = "%d V%d N%d G%s"
)
type Header struct {
Version uint64
UIDValidity uint64
NextUID uint64
GUID string
Extra map[string][]string
}
func setUint64(s string, out *uint64) error {
num, err := strconv.ParseUint(s, 10, 64)
if err != nil {
return err
}
*out = uint64(num)
return nil
}
// GenerateHeader produces a valid verison 3 dovecot-uidlist header line, using
// the time for UIDValidity and a random string for the GUID.
func GenerateHeader() (Header, error) {
// A GUID is a random 128-bit number
guidBytes := make([]byte, 16)
_, err := rand.Read(guidBytes)
if err != nil {
return Header{}, err
}
// ...hex-encoded
guidString := hex.EncodeToString(guidBytes)
return Header{
Version: uint64(3),
UIDValidity: uint64(time.Now().Unix()),
NextUID: uint64(1),
GUID: guidString,
Extra: make(map[string][]string),
}, nil
}
// ParseHeader breaks a dovecot-uidlist header line into its constituent parts
func ParseHeader(line string) (Header, error) {
var seenUIDValidity bool
var seenNextUID bool
var seenGUID bool
out := Header{}
parts := strings.Split(strings.TrimSpace(line), " ")
if len(parts) < 4 {
return out, errors.New("Short header")
}
if err := setUint64(parts[0], &out.Version); err != nil {
return out, err
}
out.Extra = make(map[string][]string)
for _, part := range parts[1:] {
part = strings.TrimSpace(part)
if len(part) == 0 {
continue
}
key := part[0:1]
value := part[1:]
switch key {
case "V":
if seenUIDValidity {
return out, errors.New("UIDVALIDITY specified twice")
}
seenUIDValidity = true
if err := setUint64(value, &out.UIDValidity); err != nil {
return out, err
}
case "N":
if seenNextUID {
return out, errors.New("NEXTUID specified twice")
}
seenNextUID = true
if err := setUint64(value, &out.NextUID); err != nil {
return out, err
}
case "G":
if seenGUID {
return out, errors.New("GUID specified twice")
}
seenGUID = true
out.GUID = value
default:
out.Extra[key] = append(out.Extra[key], value)
}
}
return out, nil
}
func (h Header) String() string {
var sb strings.Builder
fmt.Fprintf(
&sb,
HeaderTemplate,
h.Version, h.UIDValidity, h.NextUID, h.GUID,
)
for k, v := range h.Extra {
fmt.Fprintf(&sb, " %s%s", k, v)
}
return sb.String()
}

30
internal/imap/uidlist/header_test.go Normal file
View File

@@ -0,0 +1,30 @@
package uidlist_test
import (
"errors"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"ur.gs/crockery/internal/imap/uidlist"
)
func TestParseHeaderValid(t *testing.T) {
valid := " 3 V1275660208 Afortytwo N25022 G3085f01b7f11094c501100008c4a11c1 \r\n"
hdr, err := uidlist.ParseHeader(valid)
require.NoError(t, err)
assert.Equal(t, uint64(3), hdr.Version)
assert.Equal(t, uint64(1275660208), hdr.UIDValidity)
assert.Equal(t, uint64(25022), hdr.NextUID)
assert.Equal(t, "3085f01b7f11094c501100008c4a11c1", hdr.GUID)
assert.Equal(t, map[string][]string{"A": []string{"fortytwo"}}, hdr.Extra)
}
func TestParseHeaderEmpty(t *testing.T) {
hdr, err := uidlist.ParseHeader("")
require.Error(t, err, errors.New("Short header"))
assert.Equal(t, uidlist.Header{}, hdr)
}

233
internal/imap/uidlist/uidlist.go Normal file
View File

@@ -0,0 +1,233 @@
// package uidlist implements Dovecot's `dovecot-uidlist` file format for
// Maildir directories
//
// Reference: https://wiki.dovecot.org/MailboxFormat/Maildir
package uidlist
import (
"bufio"
"fmt"
"log"
"os"
"path/filepath"
"sync"
)
const (
DefaultFilename = "dovecot-uidlist"
)
type List struct {
filename string
header Header
entries []Entry
mutex *sync.Mutex
}
func Create(dir string) (*List, error) {
return CreateFile(filepath.Join(dir, DefaultFilename))
}
// FIXME: on startup, we need to scan `cur/` in case any problems arose
func Open(dir string) (*List, error) {
return OpenFile(filepath.Join(dir, DefaultFilename))
}
func CreateFile(filename string) (*List, error) {
list, err := handleFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
if err != nil {
return nil, err
}
hdr, err := GenerateHeader()
if err != nil {
return nil, err
}
list.header = hdr
// TODO: allow the dovecot-uidlist to be populated from scratch
// FIXME: in particular, NextUID and entries are wrong here
if err := list.save(); err != nil {
return nil, err
}
return list, nil
}
func OpenFile(filename string) (*List, error) {
list, err := handleFile(filename, os.O_RDWR|os.O_CREATE)
if err != nil {
return nil, err
}
if err := list.parse(); err != nil {
return nil, err
}
return list, nil
}
func handleFile(filename string, flags int) (*List, error) {
file, err := os.OpenFile(filename, flags, 0600)
if err != nil {
return nil, err
}
file.Close()
return &List{
filename: filename,
mutex: &sync.Mutex{},
}, nil
}
func (l *List) UIDValidity() uint64 {
return l.header.UIDValidity
}
func (l *List) GUID() string {
return l.header.GUID
}
func (l *List) Entries() []Entry {
l.mutex.Lock()
defer l.mutex.Unlock()
// FIXME: use copy() ?
return l.entries
}
func (l *List) EntriesInRange(start, end uint64) []Entry {
l.mutex.Lock()
defer l.mutex.Unlock()
// FIXME: this shouldn't need a full scan
var out []Entry
log.Printf("Checking range %d to %d in %#v", start, end, l.entries)
for _, e := range l.entries {
if e.UID >= start && (end == 0 || e.UID <= end) {
out = append(out, e)
}
}
return out
}
func (l *List) Entry(key string) *Entry {
l.mutex.Lock()
defer l.mutex.Unlock()
return nil
}
func (l *List) Append(filenames ...string) error {
l.mutex.Lock()
defer l.mutex.Unlock()
file, err := os.OpenFile(l.filename, os.O_APPEND|os.O_RDWR, 0600)
if err != nil {
return err
}
defer file.Close()
for _, filename := range filenames {
entry := Entry{
UID: l.header.NextUID,
LastKnownFilename: filename,
}
log.Printf("Adding entry: %#v", entry)
l.entries = append(l.entries, entry)
l.header.NextUID = l.header.NextUID + 1
// Try to write them all, at least
// FIXME: we don't bubble up errors when we should
if _, err := fmt.Fprintln(file, entry.String()); err != nil {
log.Printf("failed to write UID %d to %s: %v", entry.UID, l.filename, err)
}
}
log.Printf("Appended %d entries", len(filenames))
return nil
}
func (l *List) save() (err error) {
l.mutex.Lock()
defer l.mutex.Unlock()
// FIXME: When rewriting the file, we're meant to:
// * Take the dovecot-uidlist.lock file
// * write to a temporary file then atomically rename onto -uidlist
// Unsure if the temporary file and the lock file are the same thing or not
file, err := os.OpenFile(l.filename, os.O_TRUNC|os.O_RDWR, 0600)
if err != nil {
return
}
defer func() { err = file.Close() }()
_, err = fmt.Fprintln(file, l.header.String())
if err != nil {
return
}
for _, e := range l.entries {
_, err = fmt.Fprintln(file, e.String())
if err != nil {
return
}
}
return
}
func (l *List) parse() error {
l.mutex.Lock()
defer l.mutex.Unlock()
file, err := os.Open(l.filename)
if err != nil {
return err
}
defer file.Close()
r := bufio.NewScanner(file)
// Get the header if present, which is the first line
if !r.Scan() {
return r.Err()
}
header, err := ParseHeader(r.Text())
if err != nil {
return err
}
// Now parse each additional line
entries := []Entry{}
for r.Scan() {
entry, err := ParseEntry(r.Text())
if err != nil {
return err
}
entries = append(entries, entry)
// If NextUID is out of date in the file, ignore it. We never want to
// re-use a seen UID
if entry.UID >= l.header.NextUID {
l.header.NextUID++
}
}
l.header = header
l.entries = entries
return nil
}