Convert from BoltDB to Maildir storage for emails
This commit is contained in:
65
internal/imap/uidlist/entry.go
Normal file
65
internal/imap/uidlist/entry.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package uidlist
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Entry struct {
|
||||
UID uint64
|
||||
Extra map[string][]string
|
||||
LastKnownFilename string
|
||||
}
|
||||
|
||||
func ParseEntry(line string) (Entry, error) {
|
||||
var out Entry
|
||||
|
||||
parts := strings.SplitN(strings.TrimSpace(line), ":", 2)
|
||||
if len(parts) != 2 {
|
||||
return out, errors.New("Couldn't find start of filename")
|
||||
}
|
||||
|
||||
attrs := strings.Split(parts[0], " ")
|
||||
if len(attrs) == 0 {
|
||||
return out, errors.New("Couldn't find UID")
|
||||
}
|
||||
|
||||
if err := setUint64(attrs[0], &out.UID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
|
||||
out.Extra = make(map[string][]string)
|
||||
// FIXME: this is all disturbingly similar to header.go
|
||||
for _, attr := range attrs[1:] {
|
||||
if len(attr) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
key := attr[0:1]
|
||||
value := attr[1:]
|
||||
out.Extra[key] = append(out.Extra[key], value)
|
||||
}
|
||||
|
||||
out.LastKnownFilename = parts[1]
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (e *Entry) Key() string {
|
||||
parts := strings.SplitN(e.LastKnownFilename, ":", 2)
|
||||
|
||||
return parts[0]
|
||||
}
|
||||
|
||||
func (e *Entry) String() string {
|
||||
// FIXME: also disturbingly similar to header.go
|
||||
var sb strings.Builder
|
||||
fmt.Fprintf(&sb, "%d", e.UID)
|
||||
for k, v := range e.Extra {
|
||||
fmt.Fprintf(&sb, " %s%s", k, v)
|
||||
}
|
||||
fmt.Fprintf(&sb, " :%s", e.LastKnownFilename)
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
131
internal/imap/uidlist/header.go
Normal file
131
internal/imap/uidlist/header.go
Normal file
@@ -0,0 +1,131 @@
|
||||
package uidlist
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
HeaderTemplate = "%d V%d N%d G%s"
|
||||
)
|
||||
|
||||
type Header struct {
|
||||
Version uint64
|
||||
UIDValidity uint64
|
||||
NextUID uint64
|
||||
GUID string
|
||||
Extra map[string][]string
|
||||
}
|
||||
|
||||
func setUint64(s string, out *uint64) error {
|
||||
num, err := strconv.ParseUint(s, 10, 64)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*out = uint64(num)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GenerateHeader produces a valid verison 3 dovecot-uidlist header line, using
|
||||
// the time for UIDValidity and a random string for the GUID.
|
||||
func GenerateHeader() (Header, error) {
|
||||
// A GUID is a random 128-bit number
|
||||
guidBytes := make([]byte, 16)
|
||||
_, err := rand.Read(guidBytes)
|
||||
if err != nil {
|
||||
return Header{}, err
|
||||
}
|
||||
|
||||
// ...hex-encoded
|
||||
guidString := hex.EncodeToString(guidBytes)
|
||||
|
||||
return Header{
|
||||
Version: uint64(3),
|
||||
UIDValidity: uint64(time.Now().Unix()),
|
||||
NextUID: uint64(1),
|
||||
GUID: guidString,
|
||||
Extra: make(map[string][]string),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ParseHeader breaks a dovecot-uidlist header line into its constituent parts
|
||||
func ParseHeader(line string) (Header, error) {
|
||||
var seenUIDValidity bool
|
||||
var seenNextUID bool
|
||||
var seenGUID bool
|
||||
|
||||
out := Header{}
|
||||
parts := strings.Split(strings.TrimSpace(line), " ")
|
||||
if len(parts) < 4 {
|
||||
return out, errors.New("Short header")
|
||||
}
|
||||
|
||||
if err := setUint64(parts[0], &out.Version); err != nil {
|
||||
return out, err
|
||||
}
|
||||
|
||||
out.Extra = make(map[string][]string)
|
||||
for _, part := range parts[1:] {
|
||||
part = strings.TrimSpace(part)
|
||||
|
||||
if len(part) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
key := part[0:1]
|
||||
value := part[1:]
|
||||
switch key {
|
||||
case "V":
|
||||
if seenUIDValidity {
|
||||
return out, errors.New("UIDVALIDITY specified twice")
|
||||
}
|
||||
seenUIDValidity = true
|
||||
|
||||
if err := setUint64(value, &out.UIDValidity); err != nil {
|
||||
return out, err
|
||||
}
|
||||
case "N":
|
||||
if seenNextUID {
|
||||
return out, errors.New("NEXTUID specified twice")
|
||||
}
|
||||
seenNextUID = true
|
||||
|
||||
if err := setUint64(value, &out.NextUID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
case "G":
|
||||
if seenGUID {
|
||||
return out, errors.New("GUID specified twice")
|
||||
}
|
||||
seenGUID = true
|
||||
|
||||
out.GUID = value
|
||||
default:
|
||||
out.Extra[key] = append(out.Extra[key], value)
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (h Header) String() string {
|
||||
var sb strings.Builder
|
||||
|
||||
fmt.Fprintf(
|
||||
&sb,
|
||||
HeaderTemplate,
|
||||
h.Version, h.UIDValidity, h.NextUID, h.GUID,
|
||||
)
|
||||
|
||||
for k, v := range h.Extra {
|
||||
fmt.Fprintf(&sb, " %s%s", k, v)
|
||||
}
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
30
internal/imap/uidlist/header_test.go
Normal file
30
internal/imap/uidlist/header_test.go
Normal file
@@ -0,0 +1,30 @@
|
||||
package uidlist_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"ur.gs/crockery/internal/imap/uidlist"
|
||||
)
|
||||
|
||||
func TestParseHeaderValid(t *testing.T) {
|
||||
valid := " 3 V1275660208 Afortytwo N25022 G3085f01b7f11094c501100008c4a11c1 \r\n"
|
||||
|
||||
hdr, err := uidlist.ParseHeader(valid)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, uint64(3), hdr.Version)
|
||||
assert.Equal(t, uint64(1275660208), hdr.UIDValidity)
|
||||
assert.Equal(t, uint64(25022), hdr.NextUID)
|
||||
assert.Equal(t, "3085f01b7f11094c501100008c4a11c1", hdr.GUID)
|
||||
assert.Equal(t, map[string][]string{"A": []string{"fortytwo"}}, hdr.Extra)
|
||||
}
|
||||
|
||||
func TestParseHeaderEmpty(t *testing.T) {
|
||||
hdr, err := uidlist.ParseHeader("")
|
||||
require.Error(t, err, errors.New("Short header"))
|
||||
assert.Equal(t, uidlist.Header{}, hdr)
|
||||
}
|
||||
233
internal/imap/uidlist/uidlist.go
Normal file
233
internal/imap/uidlist/uidlist.go
Normal file
@@ -0,0 +1,233 @@
|
||||
// package uidlist implements Dovecot's `dovecot-uidlist` file format for
|
||||
// Maildir directories
|
||||
//
|
||||
// Reference: https://wiki.dovecot.org/MailboxFormat/Maildir
|
||||
package uidlist
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultFilename = "dovecot-uidlist"
|
||||
)
|
||||
|
||||
type List struct {
|
||||
filename string
|
||||
header Header
|
||||
entries []Entry
|
||||
|
||||
mutex *sync.Mutex
|
||||
}
|
||||
|
||||
func Create(dir string) (*List, error) {
|
||||
return CreateFile(filepath.Join(dir, DefaultFilename))
|
||||
}
|
||||
|
||||
// FIXME: on startup, we need to scan `cur/` in case any problems arose
|
||||
func Open(dir string) (*List, error) {
|
||||
return OpenFile(filepath.Join(dir, DefaultFilename))
|
||||
}
|
||||
|
||||
func CreateFile(filename string) (*List, error) {
|
||||
list, err := handleFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
hdr, err := GenerateHeader()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
list.header = hdr
|
||||
|
||||
// TODO: allow the dovecot-uidlist to be populated from scratch
|
||||
// FIXME: in particular, NextUID and entries are wrong here
|
||||
|
||||
if err := list.save(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return list, nil
|
||||
}
|
||||
|
||||
func OpenFile(filename string) (*List, error) {
|
||||
list, err := handleFile(filename, os.O_RDWR|os.O_CREATE)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := list.parse(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return list, nil
|
||||
}
|
||||
|
||||
func handleFile(filename string, flags int) (*List, error) {
|
||||
file, err := os.OpenFile(filename, flags, 0600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
file.Close()
|
||||
|
||||
return &List{
|
||||
filename: filename,
|
||||
mutex: &sync.Mutex{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (l *List) UIDValidity() uint64 {
|
||||
return l.header.UIDValidity
|
||||
}
|
||||
|
||||
func (l *List) GUID() string {
|
||||
return l.header.GUID
|
||||
}
|
||||
|
||||
func (l *List) Entries() []Entry {
|
||||
l.mutex.Lock()
|
||||
defer l.mutex.Unlock()
|
||||
|
||||
// FIXME: use copy() ?
|
||||
return l.entries
|
||||
}
|
||||
|
||||
func (l *List) EntriesInRange(start, end uint64) []Entry {
|
||||
l.mutex.Lock()
|
||||
defer l.mutex.Unlock()
|
||||
|
||||
// FIXME: this shouldn't need a full scan
|
||||
var out []Entry
|
||||
|
||||
log.Printf("Checking range %d to %d in %#v", start, end, l.entries)
|
||||
|
||||
for _, e := range l.entries {
|
||||
if e.UID >= start && (end == 0 || e.UID <= end) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func (l *List) Entry(key string) *Entry {
|
||||
l.mutex.Lock()
|
||||
defer l.mutex.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *List) Append(filenames ...string) error {
|
||||
l.mutex.Lock()
|
||||
defer l.mutex.Unlock()
|
||||
|
||||
file, err := os.OpenFile(l.filename, os.O_APPEND|os.O_RDWR, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer file.Close()
|
||||
|
||||
for _, filename := range filenames {
|
||||
entry := Entry{
|
||||
UID: l.header.NextUID,
|
||||
LastKnownFilename: filename,
|
||||
}
|
||||
log.Printf("Adding entry: %#v", entry)
|
||||
|
||||
l.entries = append(l.entries, entry)
|
||||
l.header.NextUID = l.header.NextUID + 1
|
||||
|
||||
// Try to write them all, at least
|
||||
// FIXME: we don't bubble up errors when we should
|
||||
if _, err := fmt.Fprintln(file, entry.String()); err != nil {
|
||||
log.Printf("failed to write UID %d to %s: %v", entry.UID, l.filename, err)
|
||||
}
|
||||
}
|
||||
log.Printf("Appended %d entries", len(filenames))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *List) save() (err error) {
|
||||
l.mutex.Lock()
|
||||
defer l.mutex.Unlock()
|
||||
|
||||
// FIXME: When rewriting the file, we're meant to:
|
||||
// * Take the dovecot-uidlist.lock file
|
||||
// * write to a temporary file then atomically rename onto -uidlist
|
||||
// Unsure if the temporary file and the lock file are the same thing or not
|
||||
|
||||
file, err := os.OpenFile(l.filename, os.O_TRUNC|os.O_RDWR, 0600)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
defer func() { err = file.Close() }()
|
||||
|
||||
_, err = fmt.Fprintln(file, l.header.String())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, e := range l.entries {
|
||||
_, err = fmt.Fprintln(file, e.String())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (l *List) parse() error {
|
||||
l.mutex.Lock()
|
||||
defer l.mutex.Unlock()
|
||||
|
||||
file, err := os.Open(l.filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer file.Close()
|
||||
|
||||
r := bufio.NewScanner(file)
|
||||
|
||||
// Get the header if present, which is the first line
|
||||
if !r.Scan() {
|
||||
return r.Err()
|
||||
}
|
||||
|
||||
header, err := ParseHeader(r.Text())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Now parse each additional line
|
||||
entries := []Entry{}
|
||||
for r.Scan() {
|
||||
entry, err := ParseEntry(r.Text())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
|
||||
// If NextUID is out of date in the file, ignore it. We never want to
|
||||
// re-use a seen UID
|
||||
if entry.UID >= l.header.NextUID {
|
||||
l.header.NextUID++
|
||||
}
|
||||
}
|
||||
|
||||
l.header = header
|
||||
l.entries = entries
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user