From 076f2c6996ab1ab861dcd03d249dc3e7e7eca4e4 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Mon, 11 Nov 2024 09:26:30 +0200 Subject: [PATCH] filtering-handler, add raw to call to addmessage. Feature: Encrypted mailboxes added --- lib/filter-handler.js | 4 +- lib/message-handler.js | 948 ++++++++++++++++++++++++----------------- 2 files changed, 562 insertions(+), 390 deletions(-) diff --git a/lib/filter-handler.js b/lib/filter-handler.js index ad9621c8..3bda7899 100644 --- a/lib/filter-handler.js +++ b/lib/filter-handler.js @@ -143,6 +143,8 @@ class FilterHandler { let prepared; + let raw = Buffer.concat(chunks, chunklen); + if (options.mimeTree) { if (options.mimeTree && options.mimeTree.header) { // remove old headers @@ -157,7 +159,6 @@ class FilterHandler { mimeTree: options.mimeTree }); } else { - let raw = Buffer.concat(chunks, chunklen); prepared = await this.prepareMessage({ raw }); @@ -661,6 +662,7 @@ class FilterHandler { date: false, flags, + raw, rawchunks }; diff --git a/lib/message-handler.js b/lib/message-handler.js index 887ab62a..a9d97efb 100644 --- a/lib/message-handler.js +++ b/lib/message-handler.js @@ -189,421 +189,488 @@ class MessageHandler { return setImmediate(() => callback(new Error('Message size ' + options.raw.length + ' bytes is too large'))); } - this.prepareMessage(options, (err, prepared) => { + this.getMailbox(options, (err, mailboxData) => { if (err) { return callback(err); } - let id = prepared.id; - let mimeTree = prepared.mimeTree; - let size = prepared.size; - let bodystructure = prepared.bodystructure; - let envelope = prepared.envelope; - let idate = prepared.idate; - let hdate = prepared.hdate; - let msgid = prepared.msgid; - let subject = prepared.subject; - let headers = prepared.headers; - - let flags = Array.isArray(options.flags) ? options.flags : [].concat(options.flags || []); - let maildata = options.maildata || this.indexer.getMaildata(mimeTree); - - this.getMailbox(options, (err, mailboxData) => { + // get target mailbox data + + options.targetMailboxEncrypted = !!mailboxData.encryptMessages; + + this.users.collection('users').findOne({ _id: options.user }, (err, userData) => { if (err) { return callback(err); } - let cleanup = (...args) => { - if (!args[0]) { - return callback(...args); - } - - let attachmentIds = Object.keys(mimeTree.attachmentMap || {}).map(key => mimeTree.attachmentMap[key]); - if (!attachmentIds.length) { - return callback(...args); - } + // get target user data - this.attachmentStorage.deleteMany(attachmentIds, maildata.magic, () => callback(...args)); - }; - - this.indexer.storeNodeBodies(maildata, mimeTree, err => { + this.prepareMessage(options, (err, prepared) => { if (err) { - return cleanup(err); + return callback(err); } - // prepare message object - let messageData = { - _id: id, + // check if already encrypted + let alreadyEncrypted = false; - // should be kept when COPY'ing or MOVE'ing - root: id, + // message already prepared, check if encrypted + const parsedHeader = (prepared.mimeTree && prepared.mimeTree?.parsedHeader) || {}; + const parsedContentType = parsedHeader['content-type']; - v: consts.SCHEMA_VERSION, + if (parsedContentType && parsedContentType.subtype === 'encrypted') { + alreadyEncrypted = true; + } - // if true then expires after rdate + retention - exp: !!mailboxData.retention, - rdate: Date.now() + (mailboxData.retention || 0), + let flags = Array.isArray(options.flags) ? options.flags : [].concat(options.flags || []); + + let addMessage = () => { + let id = prepared.id; + let mimeTree = prepared.mimeTree; + let size = prepared.size; + let bodystructure = prepared.bodystructure; + let envelope = prepared.envelope; + let idate = prepared.idate; + let hdate = prepared.hdate; + let msgid = prepared.msgid; + let subject = prepared.subject; + let headers = prepared.headers; + + let maildata = options.maildata || this.indexer.getMaildata(mimeTree); + + let cleanup = (...args) => { + if (!args[0]) { + return callback(...args); + } - // make sure the field exists. it is set to true when user is deleted - userDeleted: false, + let attachmentIds = Object.keys(mimeTree.attachmentMap || {}).map(key => mimeTree.attachmentMap[key]); + if (!attachmentIds.length) { + return callback(...args); + } - idate, - hdate, - flags, - size, + this.attachmentStorage.deleteMany(attachmentIds, maildata.magic, () => callback(...args)); + }; - // some custom metadata about the delivery - meta: options.meta || {}, + this.indexer.storeNodeBodies(maildata, mimeTree, err => { + if (err) { + return cleanup(err); + } - // list filter IDs that matched this message - filters: Array.isArray(options.filters) ? options.filters : [].concat(options.filters || []), + // prepare message object + let messageData = { + _id: id, - headers, - mimeTree, - envelope, - bodystructure, - msgid, + // should be kept when COPY'ing or MOVE'ing + root: id, - // use boolean for more commonly used (and searched for) flags - unseen: !flags.includes('\\Seen'), - flagged: flags.includes('\\Flagged'), - undeleted: !flags.includes('\\Deleted'), - draft: flags.includes('\\Draft'), + v: consts.SCHEMA_VERSION, - magic: maildata.magic, + // if true then expires after rdate + retention + exp: !!mailboxData.retention, + rdate: Date.now() + (mailboxData.retention || 0), - subject, + // make sure the field exists. it is set to true when user is deleted + userDeleted: false, - // do not archive deleted messages that have been copied - copied: false - }; + idate, + hdate, + flags, + size, - if (options.verificationResults) { - messageData.verificationResults = options.verificationResults; - } + // some custom metadata about the delivery + meta: options.meta || {}, - if (options.outbound) { - messageData.outbound = [].concat(options.outbound || []); - } + // list filter IDs that matched this message + filters: Array.isArray(options.filters) ? options.filters : [].concat(options.filters || []), - if (options.forwardTargets) { - messageData.forwardTargets = [].concat(options.forwardTargets || []); - } + headers, + mimeTree, + envelope, + bodystructure, + msgid, - if (maildata.attachments && maildata.attachments.length) { - messageData.attachments = maildata.attachments; - messageData.ha = maildata.attachments.some(a => !a.related); - } else { - messageData.ha = false; - } + // use boolean for more commonly used (and searched for) flags + unseen: !flags.includes('\\Seen'), + flagged: flags.includes('\\Flagged'), + undeleted: !flags.includes('\\Deleted'), + draft: flags.includes('\\Draft'), - if (maildata.text) { - messageData.text = maildata.text.replace(/\r\n/g, '\n').trim(); + magic: maildata.magic, - // text is indexed with a fulltext index, so only store the beginning of it - if (messageData.text.length > consts.MAX_PLAINTEXT_INDEXED) { - messageData.textFooter = messageData.text.substr(consts.MAX_PLAINTEXT_INDEXED); - messageData.text = messageData.text.substr(0, consts.MAX_PLAINTEXT_INDEXED); + subject, - // truncate remaining text if total length exceeds maximum allowed - if ( - consts.MAX_PLAINTEXT_CONTENT > consts.MAX_PLAINTEXT_INDEXED && - messageData.textFooter.length > consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED - ) { - messageData.textFooter = messageData.textFooter.substr(0, consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED); + // do not archive deleted messages that have been copied + copied: false + }; + + if (options.verificationResults) { + messageData.verificationResults = options.verificationResults; } - } - messageData.text = - messageData.text.length <= consts.MAX_PLAINTEXT_CONTENT - ? messageData.text - : messageData.text.substr(0, consts.MAX_PLAINTEXT_CONTENT); - messageData.intro = this.createIntro(messageData.text); - } + if (options.outbound) { + messageData.outbound = [].concat(options.outbound || []); + } - if (maildata.html && maildata.html.length) { - let htmlSize = 0; - messageData.html = maildata.html - .map(html => { - if (htmlSize >= consts.MAX_HTML_CONTENT || !html) { - return ''; - } + if (options.forwardTargets) { + messageData.forwardTargets = [].concat(options.forwardTargets || []); + } - if (htmlSize + Buffer.byteLength(html) <= consts.MAX_HTML_CONTENT) { - htmlSize += Buffer.byteLength(html); - return html; - } + if (maildata.attachments && maildata.attachments.length) { + messageData.attachments = maildata.attachments; + messageData.ha = maildata.attachments.some(a => !a.related); + } else { + messageData.ha = false; + } - html = html.substr(0, consts.MAX_HTML_CONTENT); - htmlSize += Buffer.byteLength(html); - return html; - }) - .filter(html => html); + if (maildata.text) { + messageData.text = maildata.text.replace(/\r\n/g, '\n').trim(); - // if message has HTML content use it instead of text/plain content for intro - messageData.intro = this.createIntro(htmlToText(messageData.html.join(''))); - } + // text is indexed with a fulltext index, so only store the beginning of it + if (messageData.text.length > consts.MAX_PLAINTEXT_INDEXED) { + messageData.textFooter = messageData.text.substr(consts.MAX_PLAINTEXT_INDEXED); + messageData.text = messageData.text.substr(0, consts.MAX_PLAINTEXT_INDEXED); - this.users.collection('users').findOneAndUpdate( - { - _id: mailboxData.user - }, - { - $inc: { - storageUsed: size - } - }, - { - returnDocument: 'after', - projection: { - storageUsed: true - } - }, - (err, r) => { - if (err) { - return cleanup(err); - } + // truncate remaining text if total length exceeds maximum allowed + if ( + consts.MAX_PLAINTEXT_CONTENT > consts.MAX_PLAINTEXT_INDEXED && + messageData.textFooter.length > consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED + ) { + messageData.textFooter = messageData.textFooter.substr(0, consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED); + } + } + messageData.text = + messageData.text.length <= consts.MAX_PLAINTEXT_CONTENT + ? messageData.text + : messageData.text.substr(0, consts.MAX_PLAINTEXT_CONTENT); - if (r && r.value) { - this.loggelf({ - short_message: '[QUOTA] +', - _mail_action: 'quota', - _user: mailboxData.user, - _inc: size, - _storage_used: r.value.storageUsed, - _sess: options.session && options.session.id, - _mailbox: mailboxData._id - }); + messageData.intro = this.createIntro(messageData.text); } - let rollback = err => { - this.users.collection('users').findOneAndUpdate( - { - _id: mailboxData.user - }, - { - $inc: { - storageUsed: -size - } - }, - { - returnDocument: 'after', - projection: { - storageUsed: true + if (maildata.html && maildata.html.length) { + let htmlSize = 0; + messageData.html = maildata.html + .map(html => { + if (htmlSize >= consts.MAX_HTML_CONTENT || !html) { + return ''; } - }, - (...args) => { - let r = args && args[1]; - - if (r && r.value) { - this.loggelf({ - short_message: '[QUOTA] -', - _mail_action: 'quota', - _user: mailboxData.user, - _inc: -size, - _storage_used: r.value.storageUsed, - _sess: options.session && options.session.id, - _mailbox: mailboxData._id, - _rollback: 'yes', - _error: err.message, - _code: err.code - }); + + if (htmlSize + Buffer.byteLength(html) <= consts.MAX_HTML_CONTENT) { + htmlSize += Buffer.byteLength(html); + return html; } - cleanup(err); - } - ); - }; + html = html.substr(0, consts.MAX_HTML_CONTENT); + htmlSize += Buffer.byteLength(html); + return html; + }) + .filter(html => html); + + // if message has HTML content use it instead of text/plain content for intro + messageData.intro = this.createIntro(htmlToText(messageData.html.join(''))); + } - // acquire new UID+MODSEQ - this.database.collection('mailboxes').findOneAndUpdate( + this.users.collection('users').findOneAndUpdate( { - _id: mailboxData._id + _id: mailboxData.user }, { $inc: { - // allocate bot UID and MODSEQ values so when journal is later sorted by - // modseq then UIDs are always in ascending order - uidNext: 1, - modifyIndex: 1 + storageUsed: size } }, { - // use original value to get correct UIDNext - returnDocument: 'before' + returnDocument: 'after', + projection: { + storageUsed: true + } }, - (err, item) => { + (err, r) => { if (err) { - return rollback(err); - } - - if (!item || !item.value) { - // was not able to acquire a lock - let err = new Error('Mailbox is missing'); - err.imapResponse = 'TRYCREATE'; - return rollback(err); - } - - let mailboxData = item.value; - - // updated message object by setting mailbox specific values - messageData.mailbox = mailboxData._id; - messageData.user = mailboxData.user; - messageData.uid = mailboxData.uidNext; - messageData.modseq = mailboxData.modifyIndex + 1; - - if (!flags.includes('\\Deleted')) { - messageData.searchable = true; + return cleanup(err); } - if (mailboxData.specialUse === '\\Junk') { - messageData.junk = true; + if (r && r.value) { + this.loggelf({ + short_message: '[QUOTA] +', + _mail_action: 'quota', + _user: mailboxData.user, + _inc: size, + _storage_used: r.value.storageUsed, + _sess: options.session && options.session.id, + _mailbox: mailboxData._id + }); } - this.getThreadId(mailboxData.user, subject, mimeTree, (err, thread) => { - if (err) { - return rollback(err); - } + let rollback = err => { + this.users.collection('users').findOneAndUpdate( + { + _id: mailboxData.user + }, + { + $inc: { + storageUsed: -size + } + }, + { + returnDocument: 'after', + projection: { + storageUsed: true + } + }, + (...args) => { + let r = args && args[1]; + + if (r && r.value) { + this.loggelf({ + short_message: '[QUOTA] -', + _mail_action: 'quota', + _user: mailboxData.user, + _inc: -size, + _storage_used: r.value.storageUsed, + _sess: options.session && options.session.id, + _mailbox: mailboxData._id, + _rollback: 'yes', + _error: err.message, + _code: err.code + }); + } - messageData.thread = thread; + cleanup(err); + } + ); + }; - this.database.collection('messages').insertOne(messageData, { writeConcern: 'majority' }, (err, r) => { + // acquire new UID+MODSEQ + this.database.collection('mailboxes').findOneAndUpdate( + { + _id: mailboxData._id + }, + { + $inc: { + // allocate bot UID and MODSEQ values so when journal is later sorted by + // modseq then UIDs are always in ascending order + uidNext: 1, + modifyIndex: 1 + } + }, + { + // use original value to get correct UIDNext + returnDocument: 'before' + }, + (err, item) => { if (err) { return rollback(err); } - if (!r || !r.acknowledged) { - let err = new Error('Failed to store message [1]'); - err.responseCode = 500; - err.code = 'StoreError'; + if (!item || !item.value) { + // was not able to acquire a lock + let err = new Error('Mailbox is missing'); + err.imapResponse = 'TRYCREATE'; return rollback(err); } - let logTime = messageData.meta.time || new Date(); - if (typeof logTime === 'number') { - logTime = new Date(logTime); - } + let mailboxData = item.value; - let uidValidity = mailboxData.uidValidity; - let uid = messageData.uid; + // updated message object by setting mailbox specific values + messageData.mailbox = mailboxData._id; + messageData.user = mailboxData.user; + messageData.uid = mailboxData.uidNext; + messageData.modseq = mailboxData.modifyIndex + 1; - if ( - options.session && - options.session.selected && - options.session.selected.mailbox && - options.session.selected.mailbox.toString() === mailboxData._id.toString() - ) { - options.session.writeStream.write(options.session.formatResponse('EXISTS', messageData.uid)); + if (!flags.includes('\\Deleted')) { + messageData.searchable = true; } - let updateAddressRegister = next => { - let addresses = []; + if (mailboxData.specialUse === '\\Junk') { + messageData.junk = true; + } - if (messageData.junk || flags.includes('\\Draft')) { - // skip junk and draft messages - return next(); + this.getThreadId(mailboxData.user, subject, mimeTree, (err, thread) => { + if (err) { + return rollback(err); } - let parsed = messageData.mimeTree && messageData.mimeTree.parsedHeader; + messageData.thread = thread; - if (parsed) { - let keyList = mailboxData.specialUse === '\\Sent' ? ['to', 'cc', 'bcc'] : ['from']; + this.database.collection('messages').insertOne(messageData, { writeConcern: 'majority' }, (err, r) => { + if (err) { + return rollback(err); + } - for (const disallowedHeader of DISALLOWED_HEADERS_FOR_ADDRESS_REGISTER) { - // if email contains headers that we do not want, - // don't add any emails to address register - if (parsed[disallowedHeader]) { + if (!r || !r.acknowledged) { + let err = new Error('Failed to store message [1]'); + err.responseCode = 500; + err.code = 'StoreError'; + return rollback(err); + } + + let logTime = messageData.meta.time || new Date(); + if (typeof logTime === 'number') { + logTime = new Date(logTime); + } + + let uidValidity = mailboxData.uidValidity; + let uid = messageData.uid; + + if ( + options.session && + options.session.selected && + options.session.selected.mailbox && + options.session.selected.mailbox.toString() === mailboxData._id.toString() + ) { + options.session.writeStream.write(options.session.formatResponse('EXISTS', messageData.uid)); + } + + let updateAddressRegister = next => { + let addresses = []; + + if (messageData.junk || flags.includes('\\Draft')) { + // skip junk and draft messages return next(); } - } - for (let key of keyList) { - if (parsed[key] && parsed[key].length) { - for (let addr of parsed[key]) { - if (/no-?reply/i.test(addr.address)) { - continue; + let parsed = messageData.mimeTree && messageData.mimeTree.parsedHeader; + + if (parsed) { + let keyList = mailboxData.specialUse === '\\Sent' ? ['to', 'cc', 'bcc'] : ['from']; + + for (const disallowedHeader of DISALLOWED_HEADERS_FOR_ADDRESS_REGISTER) { + // if email contains headers that we do not want, + // don't add any emails to address register + if (parsed[disallowedHeader]) { + return next(); } - if (!addresses.some(a => a.address === addr.address)) { - addresses.push(addr); + } + + for (let key of keyList) { + if (parsed[key] && parsed[key].length) { + for (let addr of parsed[key]) { + if (/no-?reply/i.test(addr.address)) { + continue; + } + if (!addresses.some(a => a.address === addr.address)) { + addresses.push(addr); + } + } } } } - } - } - - if (!addresses.length) { - return next(); - } - this.updateAddressRegister(mailboxData.user, addresses) - .then(() => next()) - .catch(err => next(err)); - }; + if (!addresses.length) { + return next(); + } - updateAddressRegister(() => { - this.notifier.addEntries( - mailboxData, - { - command: 'EXISTS', - uid: messageData.uid, - ignore: options.session && options.session.id, - message: messageData._id, - modseq: messageData.modseq, - unseen: messageData.unseen, - idate: messageData.idate, - thread: messageData.thread - }, - () => { - this.notifier.fire(mailboxData.user); - - let raw = options.rawchunks || options.raw; - let processAudits = async () => { - let audits = await this.database - .collection('audits') - .find({ user: mailboxData.user, expires: { $gt: new Date() } }) - .toArray(); - - let now = new Date(); - for (let auditData of audits) { - if ((auditData.start && auditData.start > now) || (auditData.end && auditData.end < now)) { - // audit not active - continue; - } - await this.auditHandler.store(auditData._id, raw, { - date: messageData.idate, - msgid: messageData.msgid, - header: messageData.mimeTree && messageData.mimeTree.parsedHeader, - ha: messageData.ha, - mailbox: mailboxData._id, - mailboxPath: mailboxData.path, - info: Object.assign({ queueId: messageData.outbound }, messageData.meta) - }); - } - }; + this.updateAddressRegister(mailboxData.user, addresses) + .then(() => next()) + .catch(err => next(err)); + }; - let next = () => { - cleanup(null, true, { - uidValidity, - uid, - id: messageData._id.toString(), - mailbox: mailboxData._id.toString(), - mailboxPath: mailboxData.path, - size, - status: 'new' - }); - }; + updateAddressRegister(() => { + this.notifier.addEntries( + mailboxData, + { + command: 'EXISTS', + uid: messageData.uid, + ignore: options.session && options.session.id, + message: messageData._id, + modseq: messageData.modseq, + unseen: messageData.unseen, + idate: messageData.idate, + thread: messageData.thread + }, + () => { + this.notifier.fire(mailboxData.user); - // do not use more suitable .finally() as it is not supported in Node v8 - return processAudits().then(next).catch(next); - } - ); + let raw = options.rawchunks || options.raw; + let processAudits = async () => { + let audits = await this.database + .collection('audits') + .find({ user: mailboxData.user, expires: { $gt: new Date() } }) + .toArray(); + + let now = new Date(); + for (let auditData of audits) { + if ( + (auditData.start && auditData.start > now) || + (auditData.end && auditData.end < now) + ) { + // audit not active + continue; + } + await this.auditHandler.store(auditData._id, raw, { + date: messageData.idate, + msgid: messageData.msgid, + header: messageData.mimeTree && messageData.mimeTree.parsedHeader, + ha: messageData.ha, + mailbox: mailboxData._id, + mailboxPath: mailboxData.path, + info: Object.assign({ queueId: messageData.outbound }, messageData.meta) + }); + } + }; + + let next = () => { + cleanup(null, true, { + uidValidity, + uid, + id: messageData._id.toString(), + mailbox: mailboxData._id.toString(), + mailboxPath: mailboxData.path, + size, + status: 'new' + }); + }; + + // do not use more suitable .finally() as it is not supported in Node v8 + return processAudits().then(next).catch(next); + } + ); + }); + }); }); - }); - }); + } + ); } ); + }); + }; + + if (!alreadyEncrypted) { + // not already encrypted, check if user has encryption on or target mailbox is encrypted + if ((userData.encryptMessages || !!mailboxData.encryptMessages) && userData.pubKey && !flags.includes('\\Draft')) { + // user has encryption on or target mailbox encrypted, encrypt message and prepare again + // do not encrypt drafts + this.encryptMessage(userData.pubKey, options.raw, (err, res) => { + if (err) { + return callback(err); + } + + if (res) { + // new encrypted raw available + options.raw = res; + } + + delete options.prepared; // delete any existing prepared as new will be generated + this.prepareMessage(options, (err, newPrepared) => { + if (err) { + return callback(err); + } + + newPrepared.id = prepared.id; // retain original + + options.prepared = newPrepared; // new prepared in options just in case + prepared = newPrepared; // overwrite top-level original prepared + options.maildata = this.indexer.getMaildata(newPrepared.mimeTree); // get new maildata of encrypted message + addMessage(); + }); + }); + } else { + // not already encrypted and no need to + addMessage(); } - ); + } else { + // message already encrypted + addMessage(); + } }); }); }); @@ -1043,82 +1110,185 @@ class MessageHandler { } } - this.database.collection('messages').insertOne(message, { writeConcern: 'majority' }, (err, r) => { - if (err) { - return cursor.close(() => done(err)); - } + const updateMessage = () => { + this.database.collection('messages').insertOne(message, { writeConcern: 'majority' }, (err, r) => { + if (err) { + return cursor.close(() => done(err)); + } - if (!r || !r.acknowledged) { - let err = new Error('Failed to store message [2]'); - err.responseCode = 500; - err.code = 'StoreError'; - return cursor.close(() => done(err)); - } + if (!r || !r.acknowledged) { + let err = new Error('Failed to store message [2]'); + err.responseCode = 500; + err.code = 'StoreError'; + return cursor.close(() => done(err)); + } - let insertId = r.insertedId; - - // delete old message - this.database.collection('messages').deleteOne( - { - _id: messageId, - mailbox: mailboxData._id, - uid: messageUid - }, - { writeConcern: 'majority' }, - (err, r) => { - if (err) { - return cursor.close(() => done(err)); - } + let insertId = r.insertedId; - if (r && r.deletedCount) { - if (options.session) { - options.session.writeStream.write(options.session.formatResponse('EXPUNGE', sourceUid)); + // delete old message + this.database.collection('messages').deleteOne( + { + _id: messageId, + mailbox: mailboxData._id, + uid: messageUid + }, + { writeConcern: 'majority' }, + (err, r) => { + if (err) { + return cursor.close(() => done(err)); } - removeEntries.push({ - command: 'EXPUNGE', - ignore: options.session && options.session.id, - uid: messageUid, - message: messageId, - unseen, - // modseq is needed to avoid updating mailbox entry - modseq: newModseq - }); + if (r && r.deletedCount) { + if (options.session) { + options.session.writeStream.write(options.session.formatResponse('EXPUNGE', sourceUid)); + } + + removeEntries.push({ + command: 'EXPUNGE', + ignore: options.session && options.session.id, + uid: messageUid, + message: messageId, + unseen, + // modseq is needed to avoid updating mailbox entry + modseq: newModseq + }); + + if (options.showExpunged) { + options.session.writeStream.write(options.session.formatResponse('EXPUNGE', messageUid)); + } + } - if (options.showExpunged) { - options.session.writeStream.write(options.session.formatResponse('EXPUNGE', messageUid)); + let entry = { + command: 'EXISTS', + uid: uidNext, + message: insertId, + unseen: message.unseen, + idate: message.idate, + thread: message.thread + }; + if (junk) { + entry.junk = junk; + } + existsEntries.push(entry); + + if (existsEntries.length >= consts.BULK_BATCH_SIZE) { + // mark messages as deleted from old mailbox + return this.notifier.addEntries(mailboxData, removeEntries, () => { + // mark messages as added to new mailbox + this.notifier.addEntries(targetData, existsEntries, () => { + removeEntries = []; + existsEntries = []; + this.notifier.fire(mailboxData.user); + processNext(); + }); + }); } + processNext(); } - - let entry = { - command: 'EXISTS', - uid: uidNext, - message: insertId, - unseen: message.unseen, - idate: message.idate, - thread: message.thread - }; - if (junk) { - entry.junk = junk; + ); + }); + }; + + if (targetData.encryptMessages) { + // move target mailbox is encrypted + const parsedHeader = (message.mimeTree && message.mimeTree.parsedHeader) || {}; + const parsedContentType = parsedHeader['content-type']; + + if (parsedContentType && parsedContentType.subtype === 'encrypted') { + // message already encrypted, just continue move + updateMessage(); + } else { + // not yet encrypted + this.users.collection('users').findOne({ _id: mailboxData.user }, (err, res) => { + if (err) { + return done(err); } - existsEntries.push(entry); - - if (existsEntries.length >= consts.BULK_BATCH_SIZE) { - // mark messages as deleted from old mailbox - return this.notifier.addEntries(mailboxData, removeEntries, () => { - // mark messages as added to new mailbox - this.notifier.addEntries(targetData, existsEntries, () => { - removeEntries = []; - existsEntries = []; - this.notifier.fire(mailboxData.user); - processNext(); + // get user data + + // get raw from existing mimetree + const outputStream = this.indexer.rebuild(message.mimeTree).value; // get raw rebuilder stream + let raw = Buffer.from([], 'binary'); // set initial raw + outputStream + .on('data', data => { + raw = Buffer.concat([raw, data]); + }) + .on('end', () => { + // when done rebuilding + this.encryptMessage(res.pubKey || '', raw, (err, res) => { + if (err) { + return done(err); + } + + // encrypt rebuilt raw + + if (res) { + // encrypted + this.prepareMessage({ raw: res }, (err, prepared) => { + if (err) { + return done(err); + } + // prepare new message structure from encrypted raw + + prepared.id = message.id; // reuse existing id + + const maildata = this.indexer.getMaildata(prepared.mimeTree); // get new maildata + + // add attachments of encrypted messages + if (maildata.attachments && maildata.attachments.length) { + message.attachments = maildata.attachments; + message.ha = maildata.attachments.some(a => !a.related); + } else { + message.ha = false; + } + + // remove fields that may leak data in FE or DB + delete message.text; + delete message.html; + message.intro = ''; + + this.indexer.storeNodeBodies(maildata, prepared.mimeTree, err => { + // store new attachments + let cleanup = (...args) => { + if (!args[0]) { + return callback(...args); + } + + let attachmentIds = Object.keys(prepared.mimeTree.attachmentMap || {}).map( + key => prepared.mimeTree.attachmentMap[key] + ); + if (!attachmentIds.length) { + return callback(...args); + } + + this.attachmentStorage.deleteMany(attachmentIds, maildata.magic, () => + callback(...args) + ); + }; + + if (err) { + return cleanup(err); + } + + // overwrite required values of existing message with new values + message.mimeTree = prepared.mimeTree; + message.size = prepared.size; + message.bodystructure = prepared.bodystructure; + message.envelope = prepared.envelope; + message.headers = prepared.headers; + updateMessage(); + }); + }); + } else { + updateMessage(); + } }); }); - } - processNext(); - } - ); - }); + }); + } + } else { + // move target is not encrypted so proceed + updateMessage(); + } } ); });