MailSend/test/maildir-sync.test.js

import assert from 'node:assert/strict';
import { mkdtempSync } from 'node:fs';
import { mkdir, rename, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { test } from 'node:test';

import {
  commitInboundMaildirMigrationStaging,
  createDomain,
  createInboundMailbox,
  createInboundMessage,
  createUser,
  createWebhook,
  getInboundMessage,
  initDatabase,
  listInboundMailboxFolders,
  listInboundMaildirIndex,
  listInboundMaildirMigrationStaging,
  listInboundMessages,
  listWebhookDeliveries,
  stageInboundMessageMaildirStorageBatch
} from '../src/db.js';
import {
  ensureMaildirMailbox,
  maildirFolderPath,
  scanMaildirMailbox,
  writeMaildirMessage
} from '../src/maildir-store.js';
import {
  acquireMaildirReconciliationLocks,
  migrateInboundMessagesToMaildir,
  reconcileMaildirMailbox
} from '../src/maildir-sync.js';

test('SQLite migration is resumable and Maildir changes reconcile into the management index', async () => {
  const dataDir = mkdtempSync(path.join(tmpdir(), 'mailhub-maildir-sync-db-'));
  const maildirRoot = path.join(dataDir, 'maildir');
  initDatabase(dataDir, 'maildir-sync-secret');
  const user = createUser({
    username: 'maildir-sync-user',
    email: 'maildir-sync-user@example.com',
    password: 'password123'
  });
  createDomain(user.id, {
    domain: 'maildir-sync.example',
    selector: 'mh',
    verificationToken: 'verify',
    dkimPublic: 'public',
    dkimPrivate: 'private',
    senderHost: 'mail.maildir-sync.example',
    sendingIp: '192.0.2.70',
    spfExtra: '',
    dmarcPolicy: 'none',
    dmarcRua: ''
  });
  const mailbox = createInboundMailbox(user.id, {
    address: 'box@maildir-sync.example',
    password: 'mailbox-pass-123'
  });
  const originalRaw = 'From: alice@example.net\r\nSubject: Existing message\r\n\r\nBody';
  const original = createInboundMessage(mailbox, {
    sender: 'alice@example.net',
    recipients: [mailbox.address],
    subject: 'Existing message',
    rawMessage: originalRaw,
    textBody: 'Body'
  });

  const interruptedStorage = await writeMaildirMessage({
    root: maildirRoot,
    address: mailbox.address,
    rawMessageBytes: Buffer.from(originalRaw, 'utf8'),
    storageKey: `mhdb-${original.id}`,
    durable: false
  });
  stageInboundMessageMaildirStorageBatch([{ id: original.id, storage: interruptedStorage }]);

  const first = await migrateInboundMessagesToMaildir({ root: maildirRoot });
  assert.deepEqual(first, { processed: 1, written: 1, reused: 0, lastId: original.id });
  const migratedFiles = await scanMaildirMailbox({ root: maildirRoot, address: mailbox.address });
  assert.equal(migratedFiles.length, 1);
  assert.equal(migratedFiles[0].storageKey, `mhdb-${original.id}`);
  assert.equal(listInboundMaildirIndex(mailbox.id)[0].storageKey, `mhdb-${original.id}`);

  const second = await migrateInboundMessagesToMaildir({ root: maildirRoot });
  assert.deepEqual(second, { processed: 0, written: 0, reused: 0, lastId: 0 });
  assert.equal((await scanMaildirMailbox({ root: maildirRoot, address: mailbox.address })).length, 1);

  const archiveCur = path.join(maildirFolderPath(maildirRoot, mailbox.address, 'Archive'), 'cur');
  await mkdir(archiveCur, { recursive: true });
  const archivedPath = path.join(
    archiveCur,
    `${migratedFiles[0].baseName}:2,S`
  );
  await rename(migratedFiles[0].filePath, archivedPath);
  const missingCounts = new Map();
  const moved = await reconcileMaildirMailbox({ root: maildirRoot, mailbox, missingCounts });
  assert.equal(moved.updated, 1);
  assert.equal(getInboundMessage(user.id, original.id).folder, 'Archive');
  assert.equal(getInboundMessage(user.id, original.id).read, true);

  await writeMaildirMessage({
    root: maildirRoot,
    address: mailbox.address,
    rawMessageBytes: Buffer.from('From: bob@example.net\r\nSubject: Appended message\r\n\r\nNew body'),
    folder: 'Sent',
    read: true,
    storageKey: 'mhappend-client-1'
  });
  const appended = await reconcileMaildirMailbox({ root: maildirRoot, mailbox, missingCounts });
  assert.equal(appended.created, 1);
  const messages = listInboundMessages(user.id, { folder: 'Sent' });
  assert.equal(messages.length, 1);
  assert.equal(messages[0].subject, 'Appended message');

  await ensureMaildirMailbox(maildirRoot, mailbox.address, ['team.ops']);
  const folderAdded = await reconcileMaildirMailbox({ root: maildirRoot, mailbox, missingCounts });
  assert.equal(folderAdded.foldersCreatedOrRestored, 1);
  assert.equal(
    listInboundMailboxFolders(user.id, mailbox.id).some((folder) => folder.name === 'team.ops'),
    true
  );
  await rm(maildirFolderPath(maildirRoot, mailbox.address, 'team.ops'), { recursive: true });
  const folderRemoved = await reconcileMaildirMailbox({ root: maildirRoot, mailbox, missingCounts });
  assert.equal(folderRemoved.foldersDeleted, 1);
  assert.equal(
    listInboundMailboxFolders(user.id, mailbox.id).some((folder) => folder.name === 'team.ops'),
    false
  );

  const archiveEntry = (await scanMaildirMailbox({ root: maildirRoot, address: mailbox.address }))
    .find((entry) => entry.storageKey === `mhdb-${original.id}`);
  await rm(archiveEntry.filePath);
  const firstMissing = await reconcileMaildirMailbox({ root: maildirRoot, mailbox, missingCounts });
  assert.equal(firstMissing.deleted, 0);
  const secondMissing = await reconcileMaildirMailbox({ root: maildirRoot, mailbox, missingCounts });
  assert.equal(secondMissing.deleted, 1);
  assert.equal(getInboundMessage(user.id, original.id), null);
});

test('Maildir reconciliation waits for sorted mailbox locks and releases them idempotently', async () => {
  const dataDir = mkdtempSync(path.join(tmpdir(), 'mailhub-maildir-lock-db-'));
  const maildirRoot = path.join(dataDir, 'maildir');
  initDatabase(dataDir, 'maildir-lock-secret');
  const user = createUser({
    username: 'maildir-lock-user',
    email: 'maildir-lock-user@example.com',
    password: 'password123'
  });
  createDomain(user.id, {
    domain: 'maildir-lock.example',
    selector: 'mh',
    verificationToken: 'verify',
    dkimPublic: 'public',
    dkimPrivate: 'private',
    senderHost: 'mail.maildir-lock.example',
    sendingIp: '192.0.2.74',
    spfExtra: '',
    dmarcPolicy: 'none',
    dmarcRua: ''
  });
  const source = createInboundMailbox(user.id, {
    address: 'source@maildir-lock.example',
    password: 'mailbox-pass-123'
  });
  const target = createInboundMailbox(user.id, {
    address: 'target@maildir-lock.example',
    password: 'mailbox-pass-456'
  });

  const releaseTransfer = await acquireMaildirReconciliationLocks([
    target.id,
    source.id,
    target.id
  ]);
  let sourceSettled = false;
  let targetSettled = false;
  const sourceReconcile = reconcileMaildirMailbox({ root: maildirRoot, mailbox: source })
    .finally(() => { sourceSettled = true; });
  const targetReconcile = reconcileMaildirMailbox({ root: maildirRoot, mailbox: target })
    .finally(() => { targetSettled = true; });
  await new Promise((resolve) => setImmediate(resolve));
  assert.equal(sourceSettled, false);
  assert.equal(targetSettled, false);
  const lockedReport = await reconcileMaildirMailbox({
    root: maildirRoot,
    mailbox: source,
    skipLock: true
  });
  assert.equal(lockedReport.mailboxId, source.id);
  assert.equal(sourceSettled, false);

  releaseTransfer();
  releaseTransfer();
  const reports = await Promise.all([sourceReconcile, targetReconcile]);
  assert.deepEqual(reports.map((report) => report.mailboxId).sort((left, right) => left - right), [
    source.id,
    target.id
  ]);

  const releaseAgain = await acquireMaildirReconciliationLocks([source.id, target.id]);
  let reversedOrderAcquired = false;
  const reversedOrder = acquireMaildirReconciliationLocks([target.id, source.id])
    .then((release) => {
      reversedOrderAcquired = true;
      return release;
    });
  await new Promise((resolve) => setImmediate(resolve));
  assert.equal(reversedOrderAcquired, false);
  releaseAgain();
  const releaseReversedOrder = await reversedOrder;
  assert.equal(reversedOrderAcquired, true);
  releaseReversedOrder();
  await assert.rejects(
    acquireMaildirReconciliationLocks([source.id, 0]),
    /邮箱锁 ID 不正确/
  );
});

test('SQLite migration stops when a resumable Maildir key contains different bytes', async () => {
  const dataDir = mkdtempSync(path.join(tmpdir(), 'mailhub-maildir-conflict-db-'));
  const maildirRoot = path.join(dataDir, 'maildir');
  initDatabase(dataDir, 'maildir-conflict-secret');
  const user = createUser({
    username: 'maildir-conflict-user',
    email: 'maildir-conflict-user@example.com',
    password: 'password123'
  });
  createDomain(user.id, {
    domain: 'maildir-conflict.example',
    selector: 'mh',
    verificationToken: 'verify',
    dkimPublic: 'public',
    dkimPrivate: 'private',
    senderHost: 'mail.maildir-conflict.example',
    sendingIp: '192.0.2.71',
    spfExtra: '',
    dmarcPolicy: 'none',
    dmarcRua: ''
  });
  const mailbox = createInboundMailbox(user.id, {
    address: 'box@maildir-conflict.example',
    password: 'mailbox-pass-123'
  });
  const preparedBeforeFailure = createInboundMessage(mailbox, {
    sender: 'first@example.net',
    recipients: [mailbox.address],
    subject: 'Prepared before conflict',
    rawMessage: 'From: first@example.net\r\nSubject: Prepared before conflict\r\n\r\nFirst'
  });
  const original = createInboundMessage(mailbox, {
    sender: 'alice@example.net',
    recipients: [mailbox.address],
    subject: 'Original message',
    rawMessage: 'From: alice@example.net\r\nSubject: Original message\r\n\r\nOriginal'
  });
  await writeMaildirMessage({
    root: maildirRoot,
    address: mailbox.address,
    rawMessageBytes: Buffer.from('From: mallory@example.net\r\nSubject: Conflict\r\n\r\nConflict'),
    storageKey: `mhdb-${original.id}`
  });

  await assert.rejects(
    migrateInboundMessagesToMaildir({ root: maildirRoot }),
    /存储标识已被其他内容占用/
  );
  assert.equal(listInboundMaildirIndex(mailbox.id).length, 0);
  const remaining = await scanMaildirMailbox({ root: maildirRoot, address: mailbox.address });
  assert.equal(remaining.some((entry) => entry.storageKey === `mhdb-${preparedBeforeFailure.id}`), false);
  assert.equal(remaining.some((entry) => entry.storageKey === `mhdb-${original.id}`), true);
});

test('Maildir cutover rejects a staging count mismatch before changing the message index', () => {
  const dataDir = mkdtempSync(path.join(tmpdir(), 'mailhub-maildir-count-db-'));
  initDatabase(dataDir, 'maildir-count-secret');
  const user = createUser({
    username: 'maildir-count-user',
    email: 'maildir-count-user@example.com',
    password: 'password123'
  });
  createDomain(user.id, {
    domain: 'maildir-count.example',
    selector: 'mh',
    verificationToken: 'verify',
    dkimPublic: 'public',
    dkimPrivate: 'private',
    senderHost: 'mail.maildir-count.example',
    sendingIp: '192.0.2.72',
    spfExtra: '',
    dmarcPolicy: 'none',
    dmarcRua: ''
  });
  const mailbox = createInboundMailbox(user.id, {
    address: 'box@maildir-count.example',
    password: 'mailbox-pass-123'
  });
  const message = createInboundMessage(mailbox, {
    sender: 'alice@example.net',
    recipients: [mailbox.address],
    subject: 'Count guard',
    rawMessage: 'From: alice@example.net\r\nSubject: Count guard\r\n\r\nBody'
  });
  stageInboundMessageMaildirStorageBatch([{
    id: message.id,
    storage: {
      backend: 'maildir',
      key: `mhdb-${message.id}`,
      relpath: `mail/new/mhdb-${message.id}.mailhub`,
      sha256: 'a'.repeat(64),
      size: 4,
      mtimeMs: Date.now(),
      indexedAt: new Date().toISOString()
    }
  }]);

  assert.throws(
    () => commitInboundMaildirMigrationStaging(2),
    /Maildir 迁移暂存数量不一致/
  );
  assert.equal(listInboundMaildirIndex(mailbox.id).length, 0);
  assert.equal(listInboundMaildirMigrationStaging().length, 1);
});

test('Maildir reconciliation restores a missing SMTP index and queues its receipt webhook once', async () => {
  const dataDir = mkdtempSync(path.join(tmpdir(), 'mailhub-maildir-webhook-db-'));
  const maildirRoot = path.join(dataDir, 'maildir');
  const database = initDatabase(dataDir, 'maildir-webhook-secret');
  const user = createUser({
    username: 'maildir-webhook-user',
    email: 'maildir-webhook-user@example.com',
    password: 'password123'
  });
  createDomain(user.id, {
    domain: 'maildir-webhook.example',
    selector: 'mh',
    verificationToken: 'verify',
    dkimPublic: 'public',
    dkimPrivate: 'private',
    senderHost: 'mail.maildir-webhook.example',
    sendingIp: '192.0.2.73',
    spfExtra: '',
    dmarcPolicy: 'none',
    dmarcRua: ''
  });
  const mailbox = createInboundMailbox(user.id, {
    address: 'box@maildir-webhook.example',
    password: 'mailbox-pass-123'
  });
  createWebhook(user.id, {
    name: 'Recovered SMTP receipt',
    url: 'https://hooks.example.com/recovered-smtp',
    events: ['received'],
    mailboxId: mailbox.id
  });
  await writeMaildirMessage({
    root: maildirRoot,
    address: mailbox.address,
    rawMessageBytes: Buffer.from([
      'Message-ID: <recovered-smtp@example.net>',
      'From: sender@example.net',
      `To: ${mailbox.address}`,
      'Subject: Recovered SMTP receipt',
      '',
      'Recovered body'
    ].join('\r\n')),
    storageKey: 'mhsmtp-recovered-1'
  });

  database.exec(`
    CREATE TRIGGER reject_recovered_smtp_webhook
    BEFORE INSERT ON webhook_deliveries
    WHEN NEW.inbound_message_id IS NOT NULL
    BEGIN
      SELECT RAISE(ABORT, 'forced webhook failure');
    END;
  `);
  await assert.rejects(
    reconcileMaildirMailbox({ root: maildirRoot, mailbox }),
    /forced webhook failure/
  );
  assert.equal(listInboundMessages(user.id, { mailboxId: mailbox.id }).length, 0);
  assert.equal(listWebhookDeliveries(user.id, { eventType: 'received' }).length, 0);
  database.exec('DROP TRIGGER reject_recovered_smtp_webhook;');

  const first = await reconcileMaildirMailbox({ root: maildirRoot, mailbox });
  assert.equal(first.created, 1);
  const [message] = listInboundMessages(user.id, { mailboxId: mailbox.id });
  assert.equal(message.subject, 'Recovered SMTP receipt');
  let deliveries = listWebhookDeliveries(user.id, { eventType: 'received' });
  assert.equal(deliveries.length, 1);
  assert.equal(deliveries[0].inboundMessageId, message.id);
  assert.equal(JSON.parse(deliveries[0].payloadJson).type, 'email.received');

  const second = await reconcileMaildirMailbox({ root: maildirRoot, mailbox });
  assert.equal(second.created, 0);
  deliveries = listWebhookDeliveries(user.id, { eventType: 'received' });
  assert.equal(deliveries.length, 1);
});