From b3d48ce9b7e9e73154140987f911499872c633b4 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Tue, 23 Apr 2024 10:48:37 +0300 Subject: [PATCH 1/8] Added submission api endpoint to api docs generation --- lib/api/submit.js | 209 +++++++++++++++++++++++----------------------- 1 file changed, 105 insertions(+), 104 deletions(-) diff --git a/lib/api/submit.js b/lib/api/submit.js index 86561551..47af61c9 100644 --- a/lib/api/submit.js +++ b/lib/api/submit.js @@ -12,8 +12,11 @@ const tools = require('../tools'); const Maildropper = require('../maildropper'); const roles = require('../roles'); const Transform = require('stream').Transform; -const { sessSchema, sessIPSchema, booleanSchema } = require('../schemas'); +const { sessSchema, sessIPSchema, booleanSchema, metaDataSchema } = require('../schemas'); const { preprocessAttachments } = require('../data-url'); +const { userId, mailboxId } = require('../schemas/request/general-schemas'); +const { AddressOptionalName, AddressOptionalNameArray, Header, Attachment, ReferenceWithAttachments } = require('../schemas/request/messages-schemas'); +const { successRes } = require('../schemas/response/general-schemas'); class StreamCollect extends Transform { constructor() { @@ -620,113 +623,111 @@ module.exports = (db, server, messageHandler, userHandler, settingsHandler) => { const submitMessageWrapper = util.promisify(submitMessage); server.post( - { name: 'send', path: '/users/:user/submit' }, + { + name: 'send', + path: '/users/:user/submit', + tags: ['Submission'], + summary: 'Submit a Message for Delivery', + description: 'Use this method to send emails from a user account', + validationObjs: { + requestBody: { + mailbox: mailboxId, + from: AddressOptionalName.description('Addres for the From: header'), + replyTo: AddressOptionalName.description('Address for the Reply-To: header'), + to: Joi.array() + .items( + Joi.object({ + name: Joi.string().empty('').max(255).description('Name of the sender'), + address: Joi.string().email({ tlds: false }).failover('').required().description('Address of the sender') + }).$_setFlag('objectName', 'AddressOptionalName') + ) + .description('Addresses for the To: header'), + + cc: AddressOptionalNameArray.description('Addresses for the Cc: header'), + + bcc: AddressOptionalNameArray.description('Addresses for the Bcc: header'), + + headers: Joi.array() + .items(Header) + .description( + 'Custom headers for the message. If reference message is set then In-Reply-To and References headers are set automatically' + ), + subject: Joi.string() + .empty('') + .max(2 * 1024) + .description('Message subject. If not then resolved from Reference message'), + text: Joi.string() + .empty('') + .max(1024 * 1024) + .description('Plaintext message'), + html: Joi.string() + .empty('') + .max(1024 * 1024) + .description('HTML formatted message'), + attachments: Joi.array().items(Attachment).description('Attachments for the message'), + + meta: metaDataSchema.label('metaData').description('Optional metadata, must be an object or JSON formatted string'), + sess: sessSchema, + ip: sessIPSchema, + reference: ReferenceWithAttachments.description( + 'Optional referenced email. If uploaded message is a reply draft and relevant fields are not provided then these are resolved from the message to be replied to' + ), + // if true then treat this message as a draft + isDraft: booleanSchema.default(false).description('Is the message a draft or not'), + // if set then this message is based on a draft that should be deleted after processing + draft: Joi.object() + .keys({ + mailbox: mailboxId, + id: Joi.number().required().description('Message ID') + }) + .description('Draft message to base this one on'), + sendTime: Joi.date().description('Send time'), + uploadOnly: booleanSchema.default(false).description('If true only uploads the message but does not send it'), + envelope: Joi.object() + .keys({ + from: AddressOptionalName.description('Addres for the From: header'), + to: Joi.array().items( + Joi.object() + .keys({ + name: Joi.string().empty('').max(255).description('Name of the sender'), + address: Joi.string().email({ tlds: false }).required().description('Address of the sender') + }) + .description('Addresses for the To: header') + ) + }) + .description('Optional envelope') + }, + queryParams: {}, + pathParams: { + user: userId + }, + response: { + 200: { + description: 'Success', + model: Joi.object({ + success: successRes, + message: Joi.object({ + mailbox: Joi.string().required().description('Mailbox ID the message was stored to'), + id: Joi.number().description('Message ID in the Mailbox').required(), + queueId: Joi.string().required().description('Queue ID in MTA') + }) + .required() + .description('Information about submitted Message') + .$_setFlag('objectName', 'MessageWithQueueId') + }) + } + } + } + }, tools.responseWrapper(async (req, res) => { res.charSet('utf-8'); - const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required(), - - mailbox: Joi.string().hex().lowercase().length(24), - - reference: Joi.object().keys({ - mailbox: Joi.string().hex().lowercase().length(24).required(), - id: Joi.number().required(), - action: Joi.string().valid('reply', 'replyAll', 'forward').required() - }), - - // if true then treat this message as a draft - isDraft: booleanSchema.default(false), - - // if set then this message is based on a draft that should be deleted after processing - draft: Joi.object().keys({ - mailbox: Joi.string().hex().lowercase().length(24).required(), - id: Joi.number().required() - }), - - uploadOnly: booleanSchema.default(false), + const { pathParams, requestBody, queryParams } = req.route.spec.validationObjs; - sendTime: Joi.date(), - - envelope: Joi.object().keys({ - from: Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }), - to: Joi.array().items( - Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }) - ) - }), - - from: Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }), - - replyTo: Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }), - - to: Joi.array().items( - Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }) - ), - - cc: Joi.array().items( - Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }) - ), - - bcc: Joi.array().items( - Joi.object().keys({ - name: Joi.string().empty('').max(255), - address: Joi.string().email({ tlds: false }).required() - }) - ), - - headers: Joi.array().items( - Joi.object().keys({ - key: Joi.string().empty('').max(255), - value: Joi.string() - .empty('') - .max(100 * 1024) - }) - ), - - subject: Joi.string() - .empty('') - .max(2 * 1024), - text: Joi.string() - .empty('') - .max(1024 * 1024), - html: Joi.string() - .empty('') - .max(1024 * 1024), - - attachments: Joi.array().items( - Joi.object().keys({ - filename: Joi.string().empty('').max(255), - - contentType: Joi.string().empty('').max(255), - contentTransferEncoding: Joi.string().empty('').trim().lowercase(), - contentDisposition: Joi.string().empty('').trim().lowercase().valid('inline', 'attachment'), - cid: Joi.string().empty('').max(255), - - encoding: Joi.string().empty('').default('base64'), - content: Joi.string().required() - }) - ), - meta: Joi.object().unknown(true), - sess: sessSchema, - ip: sessIPSchema + const schema = Joi.object({ + ...pathParams, + ...requestBody, + ...queryParams }); // extract embedded attachments from HTML From 6a9bae96b9b472c9b290d89c691b5752c81ca5ed Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Fri, 13 Sep 2024 10:12:49 +0300 Subject: [PATCH 2/8] on attachment upload calculate file content hash --- lib/attachments/gridstore-storage.js | 68 +++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/lib/attachments/gridstore-storage.js b/lib/attachments/gridstore-storage.js index 87bb5020..4498496c 100644 --- a/lib/attachments/gridstore-storage.js +++ b/lib/attachments/gridstore-storage.js @@ -7,6 +7,7 @@ const errors = require('../errors'); const log = require('npmlog'); const crypto = require('crypto'); const base64Offset = require('./base64-offset'); +const Transform = require('stream').Transform; // Set to false to disable base64 decoding feature const FEATURE_DECODE_ATTACHMENTS = true; @@ -14,6 +15,40 @@ const FEATURE_DECODE_ATTACHMENTS = true; const ORPHANED_ATTACHMENTS_DELAY = 24 * 3600 * 1000; const MAX_ORPHANED_ATTACHMENTS = 1000; +class FileHashCalculator extends Transform { + constructor(options) { + super(options); + this.bodyHash = crypto.createHash('sha256'); + this.hash = null; + } + + updateHash(chunk) { + this.bodyHash.update(chunk); + } + + _transform(chunk, encoding, callback) { + if (!chunk || !chunk.length) { + return callback(); + } + + if (typeof chunk === 'string') { + chunk = Buffer.from(chunk, encoding); + } + + this.updateHash(chunk); + this.push(chunk); + + callback(); + } + + _flush(done) { + this.hash = this.bodyHash.digest('base64'); + done(); + } +} + +let fileHashCalculator = new FileHashCalculator(); + class GridstoreStorage { constructor(options) { this.bucketName = (options.options && options.options.bucket) || 'attachments'; @@ -129,6 +164,35 @@ class GridstoreStorage { let storeLock; let attachmentCallback = (...args) => { + // store finished uploading, add the hash of the file contents to file metadata + if (args.length > 2) { + const calculatedFileContentHash = args[2]; + + this.gridfs.collection(this.bucketName + '.files').findOneAndUpdate( + { + _id: hash + }, + { + $set: { + 'metadata.fileContentHash': calculatedFileContentHash + } + }, + { + returnDocument: 'after' + }, + (err, res) => { + if (err) { + return attachmentCallback(err); + } + + if (res && res.value && res.value.metadata.fileContentHash && res.value.metadata.fileContentHash === calculatedFileContentHash) { + // all good? + // do nothing + } + } + ); + } + if (storeLock) { log.silly('GridStore', '[%s] UNLOCK lock=%s status=%s', instance, lockId, storeLock.success ? 'locked' : 'empty'); if (storeLock.success) { @@ -282,13 +346,13 @@ class GridstoreStorage { attachmentCallback(err); }); - store.once('finish', () => attachmentCallback(null, id)); + store.once('finish', () => attachmentCallback(null, id, fileHashCalculator.hash)); if (!metadata.decoded) { store.end(attachment.body); } else { let decoder = new libbase64.Decoder(); - decoder.pipe(store); + decoder.pipe(fileHashCalculator).pipe(store); decoder.once('error', err => { // pass error forward store.emit('error', err); From 2d8b782e4ec7b0c888269ffaf60166e395bc8391 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Fri, 13 Sep 2024 10:16:21 +0300 Subject: [PATCH 3/8] make stream into separate file, refactor --- lib/attachments/gridstore-storage.js | 36 ++------------------------ lib/filehash-stream.js | 38 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 34 deletions(-) create mode 100644 lib/filehash-stream.js diff --git a/lib/attachments/gridstore-storage.js b/lib/attachments/gridstore-storage.js index 4498496c..f62c3b16 100644 --- a/lib/attachments/gridstore-storage.js +++ b/lib/attachments/gridstore-storage.js @@ -7,7 +7,7 @@ const errors = require('../errors'); const log = require('npmlog'); const crypto = require('crypto'); const base64Offset = require('./base64-offset'); -const Transform = require('stream').Transform; +const FileHashCalculatorStream = require('../filehash-stream'); // Set to false to disable base64 decoding feature const FEATURE_DECODE_ATTACHMENTS = true; @@ -15,39 +15,7 @@ const FEATURE_DECODE_ATTACHMENTS = true; const ORPHANED_ATTACHMENTS_DELAY = 24 * 3600 * 1000; const MAX_ORPHANED_ATTACHMENTS = 1000; -class FileHashCalculator extends Transform { - constructor(options) { - super(options); - this.bodyHash = crypto.createHash('sha256'); - this.hash = null; - } - - updateHash(chunk) { - this.bodyHash.update(chunk); - } - - _transform(chunk, encoding, callback) { - if (!chunk || !chunk.length) { - return callback(); - } - - if (typeof chunk === 'string') { - chunk = Buffer.from(chunk, encoding); - } - - this.updateHash(chunk); - this.push(chunk); - - callback(); - } - - _flush(done) { - this.hash = this.bodyHash.digest('base64'); - done(); - } -} - -let fileHashCalculator = new FileHashCalculator(); +let fileHashCalculator = new FileHashCalculatorStream(); class GridstoreStorage { constructor(options) { diff --git a/lib/filehash-stream.js b/lib/filehash-stream.js new file mode 100644 index 00000000..514c1154 --- /dev/null +++ b/lib/filehash-stream.js @@ -0,0 +1,38 @@ +'use strict'; + +const Transform = require('stream').Transform; +const crypto = require('crypto'); + +class FileHashCalculatorStream extends Transform { + constructor(options) { + super(options); + this.bodyHash = crypto.createHash('sha256'); + this.hash = null; + } + + updateHash(chunk) { + this.bodyHash.update(chunk); + } + + _transform(chunk, encoding, callback) { + if (!chunk || !chunk.length) { + return callback(); + } + + if (typeof chunk === 'string') { + chunk = Buffer.from(chunk, encoding); + } + + this.updateHash(chunk); + this.push(chunk); + + callback(); + } + + _flush(done) { + this.hash = this.bodyHash.digest('base64'); + done(); + } +} + +module.exports = FileHashCalculatorStream; From e3afdec59d14018b15f60f3e07d93f3cac369f45 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Fri, 13 Sep 2024 15:44:51 +0300 Subject: [PATCH 4/8] create stream during the try to store. otherwise getting stuck --- lib/attachments/gridstore-storage.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/attachments/gridstore-storage.js b/lib/attachments/gridstore-storage.js index f62c3b16..b859f9ef 100644 --- a/lib/attachments/gridstore-storage.js +++ b/lib/attachments/gridstore-storage.js @@ -15,8 +15,6 @@ const FEATURE_DECODE_ATTACHMENTS = true; const ORPHANED_ATTACHMENTS_DELAY = 24 * 3600 * 1000; const MAX_ORPHANED_ATTACHMENTS = 1000; -let fileHashCalculator = new FileHashCalculatorStream(); - class GridstoreStorage { constructor(options) { this.bucketName = (options.options && options.options.bucket) || 'attachments'; @@ -191,6 +189,8 @@ class GridstoreStorage { return; } + let fileHashCalculator = new FileHashCalculatorStream(); + this.gridfs.collection(this.bucketName + '.files').findOneAndUpdate( { _id: hash From 6515b279a01cce4a1f3bcaa26ea730f6f808579a Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Mon, 16 Sep 2024 11:07:03 +0300 Subject: [PATCH 5/8] refactor file content hash update --- lib/attachments/gridstore-storage.js | 45 +++++++++++----------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/lib/attachments/gridstore-storage.js b/lib/attachments/gridstore-storage.js index b859f9ef..2bb87f62 100644 --- a/lib/attachments/gridstore-storage.js +++ b/lib/attachments/gridstore-storage.js @@ -131,34 +131,6 @@ class GridstoreStorage { let attachmentCallback = (...args) => { // store finished uploading, add the hash of the file contents to file metadata - if (args.length > 2) { - const calculatedFileContentHash = args[2]; - - this.gridfs.collection(this.bucketName + '.files').findOneAndUpdate( - { - _id: hash - }, - { - $set: { - 'metadata.fileContentHash': calculatedFileContentHash - } - }, - { - returnDocument: 'after' - }, - (err, res) => { - if (err) { - return attachmentCallback(err); - } - - if (res && res.value && res.value.metadata.fileContentHash && res.value.metadata.fileContentHash === calculatedFileContentHash) { - // all good? - // do nothing - } - } - ); - } - if (storeLock) { log.silly('GridStore', '[%s] UNLOCK lock=%s status=%s', instance, lockId, storeLock.success ? 'locked' : 'empty'); if (storeLock.success) { @@ -167,6 +139,23 @@ class GridstoreStorage { // might be already finished if retrying after delay return; } + if (args.length > 2) { + const calculatedFileContentHash = args[2]; + + this.gridfs.collection(this.bucketName + '.files').findOneAndUpdate( + { + _id: hash + }, + { + $set: { + 'metadata.fileContentHash': calculatedFileContentHash + } + }, + { + returnDocument: 'after' + } + ); + } callback(...args); }); // unset variable to prevent double releasing From daa97676ec7e2e902b8930275f02b91a59d1cbd7 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Mon, 16 Sep 2024 11:09:36 +0300 Subject: [PATCH 6/8] safer file content hash handling --- lib/attachments/gridstore-storage.js | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/attachments/gridstore-storage.js b/lib/attachments/gridstore-storage.js index 2bb87f62..fa0be0ff 100644 --- a/lib/attachments/gridstore-storage.js +++ b/lib/attachments/gridstore-storage.js @@ -142,19 +142,21 @@ class GridstoreStorage { if (args.length > 2) { const calculatedFileContentHash = args[2]; - this.gridfs.collection(this.bucketName + '.files').findOneAndUpdate( - { - _id: hash - }, - { - $set: { - 'metadata.fileContentHash': calculatedFileContentHash + if (calculatedFileContentHash) { + this.gridfs.collection(this.bucketName + '.files').findOneAndUpdate( + { + _id: hash + }, + { + $set: { + 'metadata.fileContentHash': calculatedFileContentHash + } + }, + { + returnDocument: 'after' } - }, - { - returnDocument: 'after' - } - ); + ); + } } callback(...args); }); From d195fb990488bdad83aaca13b2a305d1722b3e91 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Mon, 23 Sep 2024 20:20:48 +0300 Subject: [PATCH 7/8] refactor code. Fix possible race condition --- lib/attachments/gridstore-storage.js | 54 +++++++++++++++++----------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/lib/attachments/gridstore-storage.js b/lib/attachments/gridstore-storage.js index fa0be0ff..c9f8ff65 100644 --- a/lib/attachments/gridstore-storage.js +++ b/lib/attachments/gridstore-storage.js @@ -33,6 +33,23 @@ class GridstoreStorage { }); } + updateFileWithContentHashMetadata(args, callback, hash, calculatedFileContentHash) { + this.gridfs.collection(this.bucketName + '.files').findOneAndUpdate( + { + _id: hash + }, + { + $set: { + 'metadata.fileContentHash': calculatedFileContentHash + } + }, + { + returnDocument: 'after' + }, + () => callback(...args) // do not really care about error here. If error then highly likely the file has not been uploaded either + ); + } + async get(attachmentId) { let attachmentData = await this.gridfs.collection(this.bucketName + '.files').findOne({ _id: attachmentId @@ -131,6 +148,12 @@ class GridstoreStorage { let attachmentCallback = (...args) => { // store finished uploading, add the hash of the file contents to file metadata + let calculatedFileContentHash; + + if (args.length > 2) { + calculatedFileContentHash = args[2]; + } + if (storeLock) { log.silly('GridStore', '[%s] UNLOCK lock=%s status=%s', instance, lockId, storeLock.success ? 'locked' : 'empty'); if (storeLock.success) { @@ -139,26 +162,11 @@ class GridstoreStorage { // might be already finished if retrying after delay return; } - if (args.length > 2) { - const calculatedFileContentHash = args[2]; - - if (calculatedFileContentHash) { - this.gridfs.collection(this.bucketName + '.files').findOneAndUpdate( - { - _id: hash - }, - { - $set: { - 'metadata.fileContentHash': calculatedFileContentHash - } - }, - { - returnDocument: 'after' - } - ); - } + if (calculatedFileContentHash) { + // locked upload, new file + this.updateFileWithContentHashMetadata(args, callback, hash, calculatedFileContentHash); + return; // return from attachmentCallback. Top level callback will be ran after hash update } - callback(...args); }); // unset variable to prevent double releasing storeLock = false; @@ -170,6 +178,11 @@ class GridstoreStorage { // might be already finished if retrying after delay return; } + if (calculatedFileContentHash) { + // no lock upload, new file + this.updateFileWithContentHashMetadata(args, callback, hash, calculatedFileContentHash); + return; // return from attachmentCallback. Top level callback will be ran after hash update + } callback(...args); }; @@ -308,7 +321,8 @@ class GridstoreStorage { store.once('finish', () => attachmentCallback(null, id, fileHashCalculator.hash)); if (!metadata.decoded) { - store.end(attachment.body); + fileHashCalculator.pipe(store); + fileHashCalculator.end(attachment.body); } else { let decoder = new libbase64.Decoder(); decoder.pipe(fileHashCalculator).pipe(store); From 7b9936600488278338d72037d4c74e3031713c73 Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Thu, 26 Sep 2024 11:23:07 +0300 Subject: [PATCH 8/8] refactor function. Pass callback as last function param --- lib/attachments/gridstore-storage.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/attachments/gridstore-storage.js b/lib/attachments/gridstore-storage.js index c9f8ff65..d3fa858e 100644 --- a/lib/attachments/gridstore-storage.js +++ b/lib/attachments/gridstore-storage.js @@ -33,7 +33,7 @@ class GridstoreStorage { }); } - updateFileWithContentHashMetadata(args, callback, hash, calculatedFileContentHash) { + updateFileWithContentHashMetadata(args, hash, calculatedFileContentHash, callback) { this.gridfs.collection(this.bucketName + '.files').findOneAndUpdate( { _id: hash @@ -164,7 +164,7 @@ class GridstoreStorage { } if (calculatedFileContentHash) { // locked upload, new file - this.updateFileWithContentHashMetadata(args, callback, hash, calculatedFileContentHash); + this.updateFileWithContentHashMetadata(args, hash, calculatedFileContentHash, callback); return; // return from attachmentCallback. Top level callback will be ran after hash update } }); @@ -180,7 +180,7 @@ class GridstoreStorage { } if (calculatedFileContentHash) { // no lock upload, new file - this.updateFileWithContentHashMetadata(args, callback, hash, calculatedFileContentHash); + this.updateFileWithContentHashMetadata(args, hash, calculatedFileContentHash, callback); return; // return from attachmentCallback. Top level callback will be ran after hash update } callback(...args);