package signals import ( "context" "crypto/tls" "database/sql" "errors" "fmt" "github.com/andrew/go-computer/internal/config" crypt "github.com/andrew/go-computer/internal/crypto" "github.com/andrew/go-computer/internal/db" "github.com/andrew/go-computer/internal/netguard" "github.com/emersion/go-imap" "github.com/emersion/go-imap/client" "log" "math" "net" "os" "sort" "strings" "time" ) type SignalInput struct { ExternalID, Title, Summary string OccurredAt time.Time } type Mailbox struct { Host, Username, Password, Folder string Port int TLS bool } // AllowInsecure reports whether plaintext IMAP logins are permitted. Only // explicit development/test fixtures may set INTEGRATIONS_ALLOW_INSECURE_IMAP; // production configurations must always authenticate over verified TLS // (review NET-02). Never set it in production Compose. func AllowInsecure() bool { return os.Getenv("INTEGRATIONS_ALLOW_INSECURE_IMAP") == "1" } // checkTransport refuses password login on a plaintext socket (review NET-02). // Implicit TLS is the only production transport; there is no STARTTLS // downgrade path. func checkTransport(m Mailbox) error { if !m.TLS && !AllowInsecure() { return errors.New("signals: refusing plaintext IMAP connection; TLS is required") } return nil } // Connect dials through netguard, so user-supplied IMAP hosts cannot point // the server at internal infrastructure. The 15-second deadline covers // dial/TLS/login only and is cleared afterwards; later commands are bounded // by the caller's context (AfterFunc termination), not by a stale absolute // deadline. func Connect(ctx context.Context, m Mailbox) (*client.Client, error) { if e := checkTransport(m); e != nil { return nil, e } if m.Port == 0 { m.Port = 993 } // Keep outbound mailbox traffic on the normal IMAP ports. The fixture // flags cover loopback fakes (and deliberate LAN deployments) that listen // on an ephemeral port; production has no reason to dial, say, an SSH or // database port through the netguard escape hatch. if !netguard.AllowPrivate() && !AllowInsecure() && m.Port != 993 && m.Port != 143 { return nil, errors.New("signals: mailbox port must be 143 or 993") } if m.Folder == "" { m.Folder = "INBOX" } conn, e := netguard.DialContext(ctx, "tcp", net.JoinHostPort(m.Host, fmt.Sprint(m.Port))) if e != nil { return nil, e } if e = conn.SetDeadline(time.Now().Add(15 * time.Second)); e != nil { conn.Close() return nil, e } if m.TLS { t := tls.Client(conn, &tls.Config{ServerName: m.Host, MinVersion: tls.VersionTLS12}) if e = t.HandshakeContext(ctx); e != nil { conn.Close() return nil, e } conn = t } c, e := client.New(conn) if e != nil { conn.Close() return nil, e } if e = c.Login(m.Username, m.Password); e != nil { c.Terminate() return nil, e } if e = conn.SetDeadline(time.Time{}); e != nil { c.Terminate() return nil, e } return c, nil } func Search(ctx context.Context, m Mailbox, from, subject string) ([]SignalInput, error) { c, e := Connect(ctx, m) if e != nil { return nil, e } stop := context.AfterFunc(ctx, func() { c.Terminate() }) defer stop() defer c.Logout() return SearchConn(c, m, from, subject) } // Checkpoint is the durable per-source ingestion state (review SYNC-02): // the folder UIDVALIDITY it was recorded against and the highest UID already // considered. A zero value means "nothing ingested yet". type Checkpoint struct{ UIDValidity, LastUID uint32 } func LoadCheckpoint(ctx context.Context, q db.Queryer, sourceID string) (Checkpoint, error) { var cp Checkpoint var validity, last int64 e := q.QueryRowContext(ctx, `SELECT "uidValidity","lastUid" FROM "SignalCheckpoint" WHERE "sourceId"=$1`, sourceID).Scan(&validity, &last) if errors.Is(e, sql.ErrNoRows) { return Checkpoint{}, nil } if e != nil { return Checkpoint{}, e } cp.UIDValidity, cp.LastUID = uint32(validity), uint32(last) return cp, nil } // SaveCheckpoint persists cp. Callers must only advance a checkpoint after // the batch's signals are durably handled. The update is monotonic (review // COR-4): when uidValidity is unchanged, lastUid only advances via GREATEST, // so a slower concurrent pass can never regress the checkpoint and force a // re-scan of messages already ingested. func SaveCheckpoint(ctx context.Context, q db.Queryer, sourceID string, cp Checkpoint) error { _, e := q.ExecContext(ctx, `INSERT INTO "SignalCheckpoint"("sourceId","uidValidity","lastUid","updatedAt") VALUES($1,$2,$3,now()) ON CONFLICT("sourceId") DO UPDATE SET "uidValidity"=EXCLUDED."uidValidity","lastUid"=CASE WHEN "SignalCheckpoint"."uidValidity"=EXCLUDED."uidValidity" THEN GREATEST("SignalCheckpoint"."lastUid",EXCLUDED."lastUid") ELSE EXCLUDED."lastUid" END,"updatedAt"=now()`, sourceID, int64(cp.UIDValidity), int64(cp.LastUID)) return e } const ( // searchBatch bounds one UID FETCH round trip. searchBatch = 50 // maxIngestBatches caps a single checkpointed pass at 2500 messages per // source; a later pass continues from the persisted checkpoint. maxIngestBatches = 50 ) // SearchConn runs one bounded search on an established connection and // reports the most recent matches. It is preview semantics for interactive // checks ("Check now", legacy mailbox watch): no checkpoint is consulted or // advanced. Durable ingestion uses SearchConnFrom. An empty criteria would // match the entire mailbox, so it yields nothing instead. On a mid-pass fetch // failure it returns the matches already fetched alongside the error. func SearchConn(c *client.Client, m Mailbox, from, subject string) ([]SignalInput, error) { events, _, e := search(c, m, from, subject, nil) return events, e } // SearchConnFrom processes only messages newer than cp and returns the // checkpoint to persist after — and only after — the returned inputs are // durably handled (review SYNC-02). A mid-pass fetch failure returns the // events fetched so far; callers must persist them WITHOUT saving the // checkpoint, so the failed batch is retried and deduplicated. Batches are processed until the source // is caught up or the per-pass cap is reached. When the folder's UIDVALIDITY // changed, the checkpoint resets and the mailbox is re-scanned; Message-ID // deduplication keeps re-scans from duplicating signals that carry one. func SearchConnFrom(c *client.Client, m Mailbox, from, subject string, cp Checkpoint) ([]SignalInput, Checkpoint, error) { events, next, e := search(c, m, from, subject, &cp) return events, next, e } func search(c *client.Client, m Mailbox, from, subject string, cp *Checkpoint) ([]SignalInput, Checkpoint, error) { next := Checkpoint{} if cp != nil { next = *cp } if from == "" && subject == "" { return []SignalInput{}, next, nil } folder := m.Folder if folder == "" { folder = "INBOX" } status, e := c.Select(folder, true) if e != nil { return nil, next, e } // Effective checkpoint (review CR-15): exactly one normalized checkpoint // drives BOTH the UID filtering and the advancement below. When the // folder's UIDVALIDITY changed, the previous UID space is meaningless — // the effective checkpoint resets to zero so every UID in the new space // is (re)ingested, and filtering must never consult the stale LastUID. eff := Checkpoint{} if cp != nil { if cp.UIDValidity == status.UidValidity { eff = *cp } next = eff } next.UIDValidity = status.UidValidity criteria := imap.NewSearchCriteria() if from != "" { criteria.Header.Add("From", from) } if subject != "" { criteria.Header.Add("Subject", subject) } // Review CR-19: constrain the server-side SEARCH to the UID window that // can still hold unhandled messages, so a caught-up source does not // re-transfer its whole match history every pass. "n:*" is inclusive of // the mailbox's highest UID even when that UID is below n (RFC 3501 // seq-number semantics), so the defensive local filter below stays — // the range is an optimization, never the correctness boundary. if cp != nil && eff.LastUID > 0 && eff.LastUID < math.MaxUint32 { criteria.Uid = new(imap.SeqSet) criteria.Uid.AddRange(eff.LastUID+1, 0) // Stop 0 renders as "*" } uids, e := c.UidSearch(criteria) if e != nil { return nil, next, e } sort.Slice(uids, func(i, j int) bool { return uids[i] < uids[j] }) // truncated reports whether the per-pass cap shortened the UID list. // Review COR-1: when it did, the SEARCH did NOT consider every matching // message, so the high-water shortcut must stay off — otherwise UIDs past // the cap would land below the new checkpoint and be dropped forever. truncated := false if cp != nil { // Incremental ingestion: everything at or below the EFFECTIVE // checkpoint has already been durably handled by an earlier pass // (or the UID space was reset and nothing has — review CR-15). filtered := uids[:0] for _, uid := range uids { if uid > eff.LastUID { filtered = append(filtered, uid) } } uids = filtered if len(uids) > searchBatch*maxIngestBatches { uids = uids[:searchBatch*maxIngestBatches] truncated = true } } else if len(uids) > searchBatch { // Preview mode: the most recent matches, best effort. uids = uids[len(uids)-searchBatch:] } out := []SignalInput{} for start := 0; start < len(uids); start += searchBatch { batch := uids[start:min(start+searchBatch, len(uids))] seq := new(imap.SeqSet) seq.AddNum(batch...) ch := make(chan *imap.Message, len(batch)) done := make(chan error, 1) go func() { done <- c.UidFetch(seq, []imap.FetchItem{imap.FetchUid, imap.FetchEnvelope}, ch) }() for msg := range ch { if msg.Uid > next.LastUID { next.LastUID = msg.Uid } if msg.Envelope == nil { continue } env := msg.Envelope id := env.MessageId if id == "" { id = fmt.Sprintf("%s:%d:%d", folder, status.UidValidity, msg.Uid) } names := []string{} for _, a := range env.From { names = append(names, a.Address()) } out = append(out, SignalInput{ExternalID: id, Title: env.Subject, Summary: strings.Join(names, ", "), OccurredAt: env.Date}) } if e := <-done; e != nil { // Partial results (review follow-up): a fetch can fail after an // earlier batch (or part of this one) already produced events. // Return what was found; callers persist it but must NOT advance // the checkpoint, so the failed batch is retried and deduped. return out, next, e } } if cp != nil && !truncated && status.UidNext > next.LastUID+1 { // The SEARCH considered every message in the folder (the cap did not // shorten it), so non-matching UIDs below the mailbox high-water // mark can be skipped for good; anything arriving after this snapshot // stays above the checkpoint. Review COR-1: this shortcut is OFF // after truncation — remaining matches get a future pass instead of // being skipped forever. next.LastUID = status.UidNext - 1 } return out, next, nil } func LoadMailbox(ctx context.Context, d *db.DB, user string) (Mailbox, error) { v, e := db.One(ctx, d, `SELECT to_jsonb(m) FROM "ImapConfig" m WHERE "userId"=$1`, user) if e != nil { return Mailbox{}, e } s := func(k string) string { v, _ := v[k].(string); return v } cfg := config.Load() pw, e := crypt.Decrypt(s("password"), cfg.EncryptionSecret()) if e != nil { // Review OPS-01: rows encrypted before ENCRYPTION_KEY existed used the // auth-secret-derived key. Decrypt with the legacy key and re-encrypt // under the dedicated key so the migration completes transparently. legacy, legacyErr := crypt.Decrypt(s("password"), cfg.Secret()) if legacyErr != nil { return Mailbox{}, e } pw = legacy if cfg.EncryptionSecret() != cfg.Secret() { if enc, encErr := crypt.Encrypt(pw, cfg.EncryptionSecret()); encErr == nil { // Review CR-21: compare-and-swap against the EXACT ciphertext // this pass decrypted. A password saved concurrently between // the read and this write must win; zero updated rows means // exactly that, not permission to overwrite. res, updErr := d.ExecContext(ctx, `UPDATE "ImapConfig" SET password=$1 WHERE "userId"=$2 AND password=$3`, enc, user, s("password")) if updErr != nil { log.Printf("imap: re-encrypt stored password under ENCRYPTION_KEY: %v", updErr) } else if n, _ := res.RowsAffected(); n == 0 { log.Printf("imap: stored password changed during key migration; keeping the newer value") } } } } port, _ := v["port"].(float64) tls, _ := v["tls"].(bool) return Mailbox{s("host"), s("username"), pw, s("folder"), int(port), tls}, nil } func PublicSourceTypes() []db.Object { return []db.Object{{"type": "email", "label": "Email (IMAP)", "description": "Search your configured mailbox for matching messages", "stub": false, "configFields": []db.Object{{"key": "from", "label": "From", "placeholder": "sender@domain.com"}, {"key": "subject", "label": "Subject", "placeholder": "Subject contains"}}}} } // CheckResult is the structured outcome of one signal pass (review CR-16). // Sources counts the sources considered, Created the signals inserted, and // Failed the sources whose check did not complete successfully (mailbox // connection/search failure, signal insert failure, checkpoint failure or // status-write failure). Per-source detail is persisted on // SignalSource.lastStatus for the UI. The returned error is reserved for // pass-level infrastructure failures (loading the source list): per-source // failures live in Failed so scheduler backoff reacts to real integration // failures while a manual check still answers 200 with per-source outcomes. type CheckResult struct{ Sources, Created, Failed int } func RunSignalChecks(ctx context.Context, d *db.DB, user, bookmark string) (CheckResult, error) { res := CheckResult{} sources, e := db.Many(ctx, d, `SELECT to_jsonb(s) FROM "SignalSource" s WHERE "ownerId"=$1 AND enabled AND ($2='' OR "bookmarkId"=$2)`, user, bookmark) if e != nil { return res, e } res.Sources = len(sources) if len(sources) == 0 { return res, nil } // Load and connect once for the whole pass; every source of a user shares // the same mailbox credentials. m, connErr := LoadMailbox(ctx, d, user) var c *client.Client if connErr == nil { c, connErr = Connect(ctx, m) if connErr == nil { stop := context.AfterFunc(ctx, func() { c.Terminate() }) defer stop() defer c.Logout() } } for _, src := range sources { sourceID := fmt.Sprint(src["id"]) status := "Checked" events := []SignalInput{} e := connErr var next Checkpoint if e == nil { cfg, _ := src["config"].(map[string]any) from, _ := cfg["from"].(string) subject, _ := cfg["subject"].(string) // Incremental, checkpointed ingestion (review SYNC-02): only UIDs // newer than the durable checkpoint are processed, in bounded // batches, until the source is caught up. Review COR-3: a failed // checkpoint load is surfaced as an error for this source instead // of silently degrading to a zero checkpoint (full re-scan). var cp Checkpoint if cp, e = LoadCheckpoint(ctx, d, sourceID); e == nil { events, next, e = SearchConnFrom(c, m, from, subject, cp) } } // Persist whatever events this source produced. On a partial search // failure the caller still inserts them; the checkpoint stays put so // the failed batch is retried and ON CONFLICT dedupes the rest. insertEvents := func() error { for _, s := range events { when := s.OccurredAt if when.IsZero() { when = time.Now() } r, err := d.ExecContext(ctx, `INSERT INTO "Signal"(id,"sourceType",title,summary,"externalId","occurredAt","sourceId","bookmarkId","ownerId") VALUES($1,'email',$2,$3,$4,$5,$6,$7,$8) ON CONFLICT("sourceId","externalId") DO NOTHING`, db.NewID(), s.Title, s.Summary, s.ExternalID, when, src["id"], src["bookmarkId"], user) if err != nil { return err } n, _ := r.RowsAffected() res.Created += int(n) } return nil } if e != nil { // Connection/search failures are recorded per source (lastStatus) // and counted in Failed (review CR-16) — they are NOT folded into // the pass-level error, so a manual check still answers 200 with // the per-source outcome and db.ErrNotFound from a missing // mailbox configuration cannot surface as a misleading 404. The // scheduler reads Failed and backs off on repeated failures. status = "Error: mailbox connection or search failed" res.Failed++ // Review follow-up: a search that failed partway may still have // found real messages. Keep them instead of discarding the whole // pass; the checkpoint is deliberately not saved below. if len(events) > 0 { if err := insertEvents(); err != nil { status = "Error: signal insert failed" } } } else if err := insertEvents(); err != nil { // Review COR-5: record the per-source failure in lastStatus // and continue with the remaining sources instead of aborting // the whole pass. The checkpoint does NOT advance, so the // failed event and the rest of the batch are retried next // pass (ON CONFLICT dedupes the ones already inserted). status = "Error: signal insert failed" res.Failed++ } else if err := SaveCheckpoint(ctx, d, sourceID, next); err != nil { // Advance the checkpoint only after the batch is durably handled. status = "Error: checkpoint save failed" res.Failed++ } // Record this source's status even on error so the UI can surface it. if _, err := d.ExecContext(ctx, `UPDATE "SignalSource" SET "lastCheckedAt"=now(),"lastStatus"=$1,"updatedAt"=now() WHERE id=$2`, status, src["id"]); err != nil { res.Failed++ } } return res, nil }