ass/src/storage.ts

// https://docs.digitalocean.com/products/spaces/resources/s3-sdk-examples/
// https://www.digitalocean.com/community/tutorials/how-to-upload-a-file-to-object-storage-with-node-js
import { FileData } from './types/definitions';
import { Config, MagicNumbers } from 'ass-json'

import fs, { Stats } from 'fs-extra';
import aws from 'aws-sdk';
import Thumbnail from './thumbnails';
import Vibrant from './vibrant';
import Hash from './hash';
import { path, generateId, log } from './utils';
import { Request, Response } from 'express';
import { removeGPS } from './nightmare';
const { s3enabled, s3endpoint, s3bucket, s3usePathStyle, s3accessKey, s3secretKey, diskFilePath, saveAsOriginal, saveWithDate, savePerDay, mediaStrict, maxUploadSize }: Config = fs.readJsonSync(path('config.json'));
const { CODE_UNSUPPORTED_MEDIA_TYPE }: MagicNumbers = fs.readJsonSync(path('MagicNumbers.json'));

const ID_GEN_LENGTH = 32;
const ALLOWED_MIMETYPES = /(image)|(video)|(audio)\//;

const s3 = new aws.S3({
	s3ForcePathStyle: s3usePathStyle,
	endpoint: new aws.Endpoint(s3endpoint),
	credentials: new aws.Credentials({ accessKeyId: s3accessKey, secretAccessKey: s3secretKey })
});

function getDatedDirname() {
	if (!saveWithDate) return diskFilePath;

	// Get current month and year
	const [month, , year] = new Date().toLocaleDateString('en-US').split('/');

	// Add 0 before single digit months (6 turns into 06)
	return `${diskFilePath}${diskFilePath.endsWith('/') ? '' : '/'}${year}-${`0${month}`.slice(-2)}`; // skipcq: JS-0074
}

/**
 * A bit hacky but it works
 * @since 0.14.1
 */
function getDatedDirnameWithDay() {
	if (!savePerDay) return getDatedDirname();

	// Get current day
	const [, day] = new Date().toLocaleDateString('en-US').split('/');

	// Add 0 before single digit days (6 turns into 06)
	return `${getDatedDirname()}-${`0${day}`.slice(-2)}`; // skipcq: JS-0074
}

function getLocalFilename(req: Request) {
	let name = `${getDatedDirnameWithDay()}/${saveAsOriginal ? req.file.originalname : req.file.sha1}`;

	// Append a number if this file has already been uploaded before
	let count = 0;
	while (fs.existsSync(path(name))) {
		count++
		name = count == 1 ? name.concat(`-${count}`) : name.substring(0, name.lastIndexOf('-')).concat(`-${count}`);
	}

	return name;
}

export function processUploaded(req: Request, res: Response, next: Function) { // skipcq: JS-0045
	// Fix file object
	req.file = req.files.file;

	// Other fixes
	req.file.ext = '.'.concat((req.file.filename ?? '').split('.').pop() ?? '');
	req.file.originalname = req.file.filename ?? '';
	req.file.path = req.file.file ?? '';
	req.file.randomId = generateId('random', ID_GEN_LENGTH, 0, '');
	req.file.deleteId = generateId('random', ID_GEN_LENGTH, 0, '');

	// Set up types
	req.file.is = {
		image: false,
		video: false,
		audio: false,
		other: false
	};

	// Specify correct type
	const isType = req.file!.mimetype.includes('image') ? 'image' : req.file.mimetype.includes('video') ? 'video' : req.file.mimetype.includes('audio') ? 'audio' : 'other';
	req.file.is[isType] = true;

	// Block the resource if the mimetype is not an image or video
	if (mediaStrict && !ALLOWED_MIMETYPES.test(req.file.mimetype))
		return log
			.warn('Upload blocked', req.file.originalname, req.file.mimetype)
			.warn('Strict media mode', 'only images, videos, & audio are file permitted')
			.callback(() =>
				fs.remove(req.file.path)
					.then(() => log
						.debug('Temp file', 'deleted')
						.callback(() => res.sendStatus(CODE_UNSUPPORTED_MEDIA_TYPE)))
					.catch((err) => log
						.error('Temp file could not be deleted', err)
						.callback(() => next(err))));

	// Remove unwanted fields
	delete req.file.uuid;
	delete req.file.field;
	delete req.file.file;
	delete req.file.filename;
	delete req.file.truncated;
	delete req.file.done;

	// Temp file name used in case file already exists (long story; just don't touch this)
	let tempFileName = '';

	// Operations
	// @ts-ignore
	Promise.all([Thumbnail(req.file), Vibrant(req.file), Hash(req.file), fs.stat(req.file.path)])
		// skipcq: JS-0086
		.then(([thumbnail, vibrant, sha1, stat]: [string, string, string, Stats]) => (
			req.file.thumbnail = thumbnail, // skipcq: JS-0090
			req.file.vibrant = vibrant, // skipcq: JS-0090
			req.file.sha1 = sha1, // skipcq: JS-0090
			req.file.size = stat.size // skipcq: JS-0090
		))

		// Check if file size is too big
		.then(() => { if (req.file.size / Math.pow(1024, 2) > maxUploadSize) throw new Error('LIMIT_FILE_SIZE'); })

		// Strip EXIF data
		.then(() => removeGPS(req.file.path))
		.then((result) => log.debug('EXIF GPS data', result ? 'removed' : 'not removed'))
		.catch((err) => log.debug('!! EXIF GPS data could not be removed', err))

		// Save file
		.then(() => log.debug('Saving file', req.file.originalname, s3enabled ? 'in S3' : 'on disk'))
		.then(() =>
			// skipcq: JS-0229
			new Promise((resolve, reject) => {

				// Upload to Amazon S3
				if (s3enabled) return s3.putObject({
					Bucket: s3bucket,
					Key: req.file.randomId.concat(req.file.ext),
					ACL: 'public-read',
					ContentType: req.file.mimetype,
					Body: fs.createReadStream(req.file.path)
				}).promise().then(resolve).catch(reject);

				// Save to local storage
				else return fs.ensureDir(getDatedDirname())
					.then(() => tempFileName = getLocalFilename(req))
					.then(() => fs.copy(req.file.path, tempFileName, { preserveTimestamps: true }))
					.then(resolve).catch(reject);
			}))
		.then(() => log.debug('File saved', req.file.originalname, s3enabled ? 'in S3' : 'on disk'))
		.catch((err) => next(err))

		// Delete the file
		.then(() => fs.remove(req.file.path))
		.then(() => log.debug('Temp file', 'deleted'))

		// Fix the file path
		.then(() => !s3enabled && (req.file.path = tempFileName)) // skipcq: JS-0090
		.then(() => next())
		.catch((err) => next(err));
}

export function deleteS3(file: FileData) {
	return new Promise((resolve, reject) => s3
		.deleteObject({ Bucket: s3bucket, Key: file.randomId.concat(file.ext) })
		.promise()
		.then(resolve)
		.catch(reject));
}

function listAllKeys(resolve: Function, reject: Function, token?: string) {
	let allKeys: string[] = [];
	s3.listObjectsV2({ Bucket: s3bucket, ContinuationToken: token }).promise()
		.then((data: { [key: string]: any }) => (allKeys = allKeys.concat(data.Contents), data.IsTruncated ? listAllKeys(resolve, reject, data.NextContinuationToken) : resolve(allKeys.length))) // skipcq: JS-0086, JS-0090
		.catch((err: Error) => reject(err));
}

export function bucketSize() {
	return new Promise((resolve, reject) => (s3enabled ? listAllKeys(resolve, reject) : resolve(0)));
}