open-source-mirrors
/
ass
mirror of https://github.com/tycrek/ass
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Migrated `storage.js` to `storage.ts`

Browse Source
pull/96/head
tycrek 3 years ago
parent 7ee0080ba0
commit 55c07e0ab5
No known key found for this signature in database
GPG Key ID: 25D74F3943625263
6 changed files with 160 additions and 155 deletions
Show Stats Download Patch File Download Diff File

11
src/definitions.d.ts vendored
Unescape View File

@ -6,7 +6,15 @@ export interface User {
}
export interface FileData {
// Data from Multer file object
// Data from request file object
uuid?: string
field?: string
file?: string
filename?: string
truncated?: boolean
done?: boolean
// Old Multer data being carried over to non-Multer files
path: string
size: number
mimetype: string
@ -52,6 +60,7 @@ export interface AssRequest extends Request {
ass?: { resourceId: string }
token?: string
file?: FileData
files?: { [key: string]: any }
}
export interface AssResponse extends Response {

2
src/hash.ts
Unescape View File

@ -9,7 +9,7 @@ import { log } from './utils';
* @param {*} file The file to hash
* @returns The SHA1 hash
*/
export default (file: FileData) =>
export default (file: FileData): Promise<string> =>
new Promise((resolve, reject) =>
toArray((fs.createReadStream(file.path)))
.then((parts: any[]) => Buffer.concat(parts.map((part: any) => (Buffer.isBuffer(part) ? part : Buffer.from(part)))))

151
src/storage.js
Unescape View File

@ -1,151 +0,0 @@
// https://docs.digitalocean.com/products/spaces/resources/s3-sdk-examples/
// https://www.digitalocean.com/community/tutorials/how-to-upload-a-file-to-object-storage-with-node-js
const fs = require('fs-extra');
const aws = require('aws-sdk');
const Thumbnail = require('./thumbnails').default;
const Vibrant = require('./vibrant').default;
const Hash = require('./hash').default;
const { generateId, log } = require('./utils');
const { s3enabled, s3endpoint, s3bucket, s3usePathStyle, s3accessKey, s3secretKey, diskFilePath, saveAsOriginal, saveWithDate, mediaStrict, maxUploadSize } = require('../config.json');
const { CODE_UNSUPPORTED_MEDIA_TYPE } = require('../MagicNumbers.json');
const ID_GEN_LENGTH = 32;
const ALLOWED_MIMETYPES = /(image)|(video)|(audio)\//;
const s3 = new aws.S3({
s3ForcePathStyle: s3usePathStyle,
endpoint: new aws.Endpoint(s3endpoint),
credentials: new aws.Credentials({ accessKeyId: s3accessKey, secretAccessKey: s3secretKey })
});
function getDatedDirname() {
if (!saveWithDate) return diskFilePath;
// Get current month and year
const [month, , year] = new Date().toLocaleDateString('en-US').split('/');
// Add 0 before single digit months (6 turns into 06)
return `${diskFilePath}${diskFilePath.endsWith('/') ? '' : '/'}${year}-${`0${month}`.slice(-2)}`; // skipcq: JS-0074
}
function getLocalFilename(req) {
return `${getDatedDirname()}/${saveAsOriginal ? req.file.originalname : req.file.sha1}`;
}
function processUploaded(req, res, next) { // skipcq: JS-0045
// Fix file object
req.file = req.files.file;
// Other fixes
req.file.ext = '.'.concat(req.file.filename.split('.').pop());
req.file.originalname = req.file.filename;
req.file.path = req.file.file;
req.file.randomId = generateId('random', ID_GEN_LENGTH, null, null);
req.file.deleteId = generateId('random', ID_GEN_LENGTH, null, null);
// Set up types
req.file.is = {
image: false,
video: false,
audio: false,
other: false
};
// Specify correct type
const isType = req.file.mimetype.includes('image') ? 'image' : req.file.mimetype.includes('video') ? 'video' : req.file.mimetype.includes('audio') ? 'audio' : 'other';
req.file.is[isType] = true;
// Block the resource if the mimetype is not an image or video
if (mediaStrict && !ALLOWED_MIMETYPES.test(req.file.mimetype))
return log
.warn('Upload blocked', req.file.originalname, req.file.mimetype)
.warn('Strict media mode', 'only images, videos, & audio are file permitted')
.callback(() =>
fs.remove(req.file.path)
.then(() => log
.debug('Temp file', 'deleted')
.callback(() => res.sendStatus(CODE_UNSUPPORTED_MEDIA_TYPE)))
.catch((err) => log
.error('Temp file could not be deleted', err)
.callback(next, err)));
// Remove unwanted fields
delete req.file.uuid;
delete req.file.field;
delete req.file.file;
delete req.file.filename;
delete req.file.truncated;
delete req.file.done;
// Operations
Promise.all([Thumbnail(req.file), Vibrant(req.file), Hash(req.file), fs.stat(req.file.path)])
// skipcq: JS-0086
.then(([thumbnail, vibrant, sha1, stat]) => (
req.file.thumbnail = thumbnail, // skipcq: JS-0090
req.file.vibrant = vibrant, // skipcq: JS-0090
req.file.sha1 = sha1, // skipcq: JS-0090
req.file.size = stat.size // skipcq: JS-0090
))
// Check if file size is too big
.then(() => { if (req.file.size / Math.pow(1024, 2) > maxUploadSize) throw new Error('LIMIT_FILE_SIZE'); })
// Save file
.then(() => log.debug('Saving file', req.file.originalname, s3enabled ? 'in S3' : 'on disk'))
.then(() =>
// skipcq: JS-0229
new Promise((resolve, reject) => s3enabled
// Upload to Amazon S3
? s3.putObject({
Bucket: s3bucket,
Key: req.file.randomId.concat(req.file.ext),
ACL: 'public-read',
ContentType: req.file.mimetype,
Body: fs.createReadStream(req.file.path)
}).promise().then(resolve).catch(reject)
// Save to local storage
: fs.ensureDir(getDatedDirname())
.then(() => fs.copy(req.file.path, getLocalFilename(req), { preserveTimestamps: true }))
.then(resolve)
.catch(reject)
))
.then(() => log.debug('File saved', req.file.originalname, s3enabled ? 'in S3' : 'on disk'))
.catch(next)
// Delete the file
.then(() => fs.remove(req.file.path))
.then(() => log.debug('Temp file', 'deleted'))
// Fix the file path
.then(() => !s3enabled && (req.file.path = getLocalFilename(req))) // skipcq: JS-0090
.then(() => next())
.catch(next);
}
function deleteS3(file) {
return new Promise((resolve, reject) => s3
.deleteObject({ Bucket: s3bucket, Key: file.randomId.concat(file.ext) })
.promise()
.then(resolve)
.catch(reject));
}
function listAllKeys(resolve, reject, token) {
let allKeys = [];
s3.listObjectsV2({ Bucket: s3bucket, ContinuationToken: token }).promise()
.then((data) => (allKeys = allKeys.concat(data.Contents), data.IsTruncated ? listAllKeys(resolve, reject, data.NextContinuationToken) : resolve(allKeys.length))) // skipcq: JS-0086, JS-0090
.catch(reject);
}
function bucketSize() {
return new Promise((resolve, reject) => (s3enabled ? listAllKeys(resolve, reject) : resolve(0)));
}
module.exports = {
processUploaded,
deleteS3,
bucketSize
};

147
src/storage.ts
Unescape View File

@ -0,0 +1,147 @@
// https://docs.digitalocean.com/products/spaces/resources/s3-sdk-examples/
// https://www.digitalocean.com/community/tutorials/how-to-upload-a-file-to-object-storage-with-node-js
import { AssRequest, AssResponse, FileData } from './definitions';
import fs, { Stats } from 'fs-extra';
import aws from 'aws-sdk';
import Thumbnail from './thumbnails';
import Vibrant from './vibrant';
import Hash from './hash';
import { generateId, log } from './utils';
const { s3enabled, s3endpoint, s3bucket, s3usePathStyle, s3accessKey, s3secretKey, diskFilePath, saveAsOriginal, saveWithDate, mediaStrict, maxUploadSize } = require('../config.json');
const { CODE_UNSUPPORTED_MEDIA_TYPE } = require('../MagicNumbers.json');
const ID_GEN_LENGTH = 32;
const ALLOWED_MIMETYPES = /(image)|(video)|(audio)\//;
const s3 = new aws.S3({
s3ForcePathStyle: s3usePathStyle,
endpoint: new aws.Endpoint(s3endpoint),
credentials: new aws.Credentials({ accessKeyId: s3accessKey, secretAccessKey: s3secretKey })
});
function getDatedDirname() {
if (!saveWithDate) return diskFilePath;
// Get current month and year
const [month, , year] = new Date().toLocaleDateString('en-US').split('/');
// Add 0 before single digit months (6 turns into 06)
return `${diskFilePath}${diskFilePath.endsWith('/') ? '' : '/'}${year}-${`0${month}`.slice(-2)}`; // skipcq: JS-0074
}
function getLocalFilename(req: AssRequest) {
return `${getDatedDirname()}/${saveAsOriginal ? req.file!.originalname : req.file!.sha1}`;
}
export function processUploaded(req: AssRequest, res: AssResponse, next: Function) { // skipcq: JS-0045
// Fix file object
req.file = req.files!.file;
// Other fixes
req.file!.ext = '.'.concat((req.file!.filename ?? '').split('.').pop() ?? '');
req.file!.originalname = req.file!.filename ?? '';
req.file!.path = req.file!.file ?? '';
req.file!.randomId = generateId('random', ID_GEN_LENGTH, 0, '');
req.file!.deleteId = generateId('random', ID_GEN_LENGTH, 0, '');
// Set up types
req.file!.is = {
image: false,
video: false,
audio: false,
other: false
};
// Specify correct type
const isType = req.file!.mimetype.includes('image') ? 'image' : req.file!.mimetype.includes('video') ? 'video' : req.file!.mimetype.includes('audio') ? 'audio' : 'other';
req.file!.is[isType] = true;
// Block the resource if the mimetype is not an image or video
if (mediaStrict && !ALLOWED_MIMETYPES.test(req.file!.mimetype))
return log
.warn('Upload blocked', req.file!.originalname, req.file!.mimetype)
.warn('Strict media mode', 'only images, videos, & audio are file permitted')
.callback(() =>
fs.remove(req.file!.path)
.then(() => log
.debug('Temp file', 'deleted')
.callback(() => res.sendStatus(CODE_UNSUPPORTED_MEDIA_TYPE)))
.catch((err) => log
.error('Temp file could not be deleted', err)
.callback(next, err)));
// Remove unwanted fields
delete req.file!.uuid;
delete req.file!.field;
delete req.file!.file;
delete req.file!.filename;
delete req.file!.truncated;
delete req.file!.done;
// Operations
// @ts-ignore
Promise.all([Thumbnail(req.file), Vibrant(req.file), Hash(req.file), fs.stat(req.file!.path)])
// skipcq: JS-0086
.then(([thumbnail, vibrant, sha1, stat]: [string, string, string, Stats]) => (
req.file!.thumbnail = thumbnail, // skipcq: JS-0090
req.file!.vibrant = vibrant, // skipcq: JS-0090
req.file!.sha1 = sha1, // skipcq: JS-0090
req.file!.size = stat.size // skipcq: JS-0090
))
// Check if file size is too big
.then(() => { if (req.file!.size / Math.pow(1024, 2) > maxUploadSize) throw new Error('LIMIT_FILE_SIZE'); })
// Save file
.then(() => log.debug('Saving file', req.file!.originalname, s3enabled ? 'in S3' : 'on disk'))
.then(() =>
// skipcq: JS-0229
new Promise((resolve, reject) => s3enabled
// Upload to Amazon S3
? s3.putObject({
Bucket: s3bucket,
Key: req.file!.randomId.concat(req.file!.ext),
ACL: 'public-read',
ContentType: req.file!.mimetype,
Body: fs.createReadStream(req.file!.path)
}).promise().then(resolve).catch(reject)
// Save to local storage
: fs.ensureDir(getDatedDirname())
.then(() => fs.copy(req.file!.path, getLocalFilename(req), { preserveTimestamps: true }))
.then(resolve)
.catch(reject)
))
.then(() => log.debug('File saved', req.file!.originalname, s3enabled ? 'in S3' : 'on disk'))
.catch((err) => next(err))
// Delete the file
.then(() => fs.remove(req.file!.path))
.then(() => log.debug('Temp file', 'deleted'))
// Fix the file path
.then(() => !s3enabled && (req.file!.path = getLocalFilename(req))) // skipcq: JS-0090
.then(() => next())
.catch((err) => next(err));
}
export function deleteS3(file: FileData) {
return new Promise((resolve, reject) => s3
.deleteObject({ Bucket: s3bucket, Key: file.randomId.concat(file.ext) })
.promise()
.then(resolve)
.catch(reject));
}
function listAllKeys(resolve: Function, reject: Function, token?: string) {
let allKeys: string[] = [];
s3.listObjectsV2({ Bucket: s3bucket, ContinuationToken: token }).promise()
.then((data: { [key: string]: any }) => (allKeys = allKeys.concat(data.Contents), data.IsTruncated ? listAllKeys(resolve, reject, data.NextContinuationToken) : resolve(allKeys.length))) // skipcq: JS-0086, JS-0090
.catch((err) => reject(err));
}
export function bucketSize() {
return new Promise((resolve, reject) => (s3enabled ? listAllKeys(resolve, reject) : resolve(0)));
}

2
src/thumbnails.ts
Unescape View File

@ -82,7 +82,7 @@ function getImageThumbnail(file: FileData) {
* @param {*} file The file to generate a thumbnail for
* @returns The thumbnail filename (NOT the path)
*/
export default (file: FileData) =>
export default (file: FileData): Promise<string> =>
new Promise((resolve, reject) =>
(file.is.video ? getVideoThumbnail : file.is.image ? getImageThumbnail : () => Promise.resolve())(file)
.then(() => resolve((file.is.video || file.is.image) ? getNewName(file.randomId) : file.is.audio ? 'views/ass-audio-icon.png' : 'views/ass-file-icon.png'))

2
src/vibrant.ts
Unescape View File

@ -26,4 +26,4 @@ function getVibrant(file: FileData, resolve: Function, reject: Function) {
* @param {*} file The file to get a colour from
* @returns The Vibrant colour as a Hex value (or random Hex value for videos)
*/
export default (file: FileData) => new Promise((resolve, reject) => !file.is.image ? resolve(randomHexColour()) : getVibrant(file, resolve, reject)); // skipcq: JS-0229
export default (file: FileData): Promise<string> => new Promise((resolve, reject) => !file.is.image ? resolve(randomHexColour()) : getVibrant(file, resolve, reject)); // skipcq: JS-0229

Write Preview
Loading…
Cancel
Save