ass/src/storage.js

// https://docs.digitalocean.com/products/spaces/resources/s3-sdk-examples/
// https://www.digitalocean.com/community/tutorials/how-to-upload-a-file-to-object-storage-with-node-js

const fs = require('fs-extra');
const aws = require('aws-sdk');
const multer = require('multer');
const Thumbnail = require('./thumbnails');
const Vibrant = require('./vibrant');
const Hash = require('./hash');
const { getDatedDirname, sanitize, generateId, formatBytes, log } = require('./utils');
const { s3enabled, s3endpoint, s3bucket, s3usePathStyle, s3accessKey, s3secretKey, saveAsOriginal, maxUploadSize, mediaStrict } = require('../config.json');
const { CODE_UNSUPPORTED_MEDIA_TYPE } = require('../MagicNumbers.json');

const ID_GEN_LENGTH = 32;
const ALLOWED_MIMETYPES = /(image)|(video)|(audio)\//;

const s3 = new aws.S3({
	s3ForcePathStyle: s3usePathStyle,
	endpoint: new aws.Endpoint(s3endpoint),
	credentials: new aws.Credentials({ accessKeyId: s3accessKey, secretAccessKey: s3secretKey })
});

function saveFile(req) {
	log.null(req.file, 'Unable to save req.file!')
		.debug('Temp file saving');
	return new Promise((resolve, reject) =>
		fs.ensureDir(getDatedDirname())
			.then(() => fs.createWriteStream(req.file.path.concat('.temp')))
			.then((stream) => req.file.stream.pipe(stream).on('finish', () => log.debug('Temp file saved', req.file.path, formatBytes(req.file.size)).callback(resolve)).on('error', reject))
			.catch(reject));
}

function getLocalFilename(req) {
	return `${getDatedDirname()}/${saveAsOriginal ? req.file.originalname : req.file.sha1}`;
}

function processUploaded(req, res, next) {
	// Fixes
	req.file.mimetype = req.file.detectedMimeType || req.file.clientReportedMimeType;
	req.file.ext = req.file.detectedFileExtension || req.file.clientReportedFileExtension;
	req.file.originalname = sanitize(req.file.originalName);
	req.file.randomId = generateId('random', ID_GEN_LENGTH, null, null);
	req.file.deleteId = generateId('random', ID_GEN_LENGTH, null, null);

	// Set up types
	req.file.is = {
		image: false,
		video: false,
		audio: false,
		other: false
	};

	// Specify correct type
	const isType = req.file.mimetype.includes('image') ? 'image' : req.file.mimetype.includes('video') ? 'video' : req.file.mimetype.includes('audio') ? 'audio' : 'other';
	req.file.is[isType] = true;

	// Block the resource if the mimetype is not an image or video
	if (mediaStrict && !ALLOWED_MIMETYPES.test(req.file.mimetype)) {
		return log
			.warn('Upload blocked', req.file.originalname, req.file.mimetype)
			.warn('Strict media mode', 'only images, videos, & audio are file permitted')
			.callback(() =>
				fs.remove(req.file.path.concat('.temp'))
					.then(() => log
						.debug('Temp file', 'deleted')
						.callback(() => res.sendStatus(CODE_UNSUPPORTED_MEDIA_TYPE)))
					.catch((err) => log
						.error('Temp file could not be deleted', err)
						.callback(next, err)));
	}

	// Remove unwanted fields
	delete req.file.fieldName;
	delete req.file.originalName;
	delete req.file.clientReportedMimeType;
	delete req.file.clientReportedFileExtension;
	delete req.file.detectedMimeType;
	delete req.file.detectedFileExtension;

	// Operations
	saveFile(req)
		.then(() => req.file.path = req.file.path.concat('.temp')) // skipcq: JS-0086
		.then(() => Promise.all([Thumbnail(req.file), Vibrant(req.file), Hash(req.file)]))
		// skipcq: JS-0086
		.then(([thumbnail, vibrant, sha1]) => (
			req.file.thumbnail = thumbnail, // skipcq: JS-0090
			req.file.vibrant = vibrant, // skipcq: JS-0090
			req.file.sha1 = sha1 // skipcq: JS-0090
		))

		.then(() => log.debug('Saving file', req.file.originalname, s3enabled ? 'in S3' : 'on disk'))
		.then(() =>
			// skipcq: JS-0229
			new Promise((resolve, reject) => s3enabled

				// Upload to Amazon S3
				? s3.putObject({
					Bucket: s3bucket,
					Key: req.file.randomId.concat(req.file.ext),
					ACL: 'public-read',
					ContentType: req.file.mimetype,
					Body: fs.createReadStream(req.file.path)
				}).promise().then(resolve).catch(reject)

				// Save to local storage
				: fs.ensureDir(getDatedDirname())
					.then(() => fs.copy(req.file.path, getLocalFilename(req), { preserveTimestamps: true }))
					.then(resolve)
					.catch(reject)
			))
		.then(() => log.debug('File saved', req.file.originalname, s3enabled ? 'in S3' : 'on disk'))
		.then(() => fs.remove(req.file.path))
		.then(() => log.debug('Temp file', 'deleted'))
		.then(() => !s3enabled && (req.file.path = getLocalFilename(req))) // skipcq: JS-0090
		.then(() => delete req.file.stream)
		.then(() => next())
		.catch(next);
}

function deleteS3(file) {
	return new Promise((resolve, reject) => s3
		.deleteObject({ Bucket: s3bucket, Key: file.randomId.concat(file.ext) })
		.promise()
		.then(resolve)
		.catch(reject));
}

function listAllKeys(resolve, reject, token) {
	let allKeys = [];
	s3.listObjectsV2({ Bucket: s3bucket, ContinuationToken: token }).promise()
		.then((data) => (allKeys = allKeys.concat(data.Contents), data.IsTruncated ? listAllKeys(resolve, reject, data.NextContinuationToken) : resolve(allKeys.length))) // skipcq: JS-0086, JS-0090
		.catch(reject);
}

function bucketSize() {
	return new Promise((resolve, reject) => (s3enabled ? listAllKeys(resolve, reject) : resolve(0)));
}

module.exports = {
	doUpload: multer({ limits: { fileSize: `${maxUploadSize}MB` } }).single('file'),
	processUploaded,
	deleteS3,
	bucketSize
};