From 46285245348d927efb4a97a71ff8b82c42983bfd Mon Sep 17 00:00:00 2001 From: tycrek Date: Tue, 29 Jun 2021 17:31:33 -0600 Subject: [PATCH 1/7] finished writing new storage system, must run `npm i` to update packages --- ass.js | 53 ++---- hash.js | 9 +- package-lock.json | 406 +++++++++++++++++++++++++++++++--------------- package.json | 4 +- storage.js | 121 +++++++++----- thumbnails.js | 58 +++++-- utils.js | 13 +- vibrant.js | 33 ++-- 8 files changed, 463 insertions(+), 234 deletions(-) diff --git a/ass.js b/ass.js index e3a0116..c6ac6c3 100755 --- a/ass.js +++ b/ass.js @@ -7,7 +7,7 @@ try { } // Load the config -const { host, port, useSsl, resourceIdSize, diskFilePath, gfyIdSize, resourceIdType, isProxied, s3enabled, saveAsOriginal } = require('./config.json'); +const { host, port, useSsl, resourceIdSize, diskFilePath, gfyIdSize, resourceIdType, isProxied, s3enabled } = require('./config.json'); //#region Imports const fs = require('fs-extra'); @@ -19,11 +19,9 @@ const fetch = require('node-fetch'); const marked = require('marked'); const { DateTime } = require('luxon'); const { WebhookClient, MessageEmbed } = require('discord.js'); -const Thumbnail = require('./thumbnails'); -const Vibrant = require('./vibrant'); -const Hash = require('./hash'); -const { uploadLocal, uploadS3, deleteS3 } = require('./storage'); -const { path, saveData, log, verify, getTrueHttp, getTrueDomain, renameFile, generateToken, generateId, formatBytes, formatTimestamp, arrayEquals, getS3url, getDirectUrl, getSafeExt, getResourceColor, downloadTempS3, sanitize, replaceholder } = require('./utils'); +const { doUpload, deleteS3, processUploaded } = require('./storage'); +const { path, saveData, log, verify, getTrueHttp, getTrueDomain, generateToken, generateId, formatBytes, + formatTimestamp, arrayEquals, getS3url, getDirectUrl, getSafeExt, getResourceColor, replaceholder } = require('./utils'); const { CODE_NO_CONTENT, CODE_BAD_REQUEST, CODE_UNAUTHORIZED, CODE_NOT_FOUND } = require('./MagicNumbers.json'); //#endregion @@ -112,40 +110,8 @@ function startup() { !verify(req, users) ? res.sendStatus(CODE_UNAUTHORIZED) : next(); // skipcq: JS-0093 }); - // Generate ID's to use for other functions - app.post('/', (req, _res, next) => (req.randomId = generateId('random', 32, null, null), next())); // skipcq: JS-0074, JS-0086, JS-0090 - app.post('/', (req, _res, next) => (req.deleteId = generateId('random', 32, null, null), next())); // skipcq: JS-0074, JS-0086, JS-0090 - - // Upload file (local & S3) // skipcq: JS-0093 - s3enabled - ? app.post('/', (req, res, next) => uploadS3(req, res, (error) => ((error) && console.error(error), next()))) // skipcq: JS-0090 - : app.post('/', uploadLocal, ({ next }) => next()); - - // Pre-response operations - app.post('/', (req, _res, next) => { - req.file.randomId = req.randomId; - req.file.deleteId = req.deleteId; - - // Sanitize filename just in case Multer didn't catch it - req.file.originalname = sanitize(req.file.originalname); - - // Download a temp copy to work with if using S3 storage - (s3enabled ? downloadTempS3(req.file) : new Promise((resolve) => resolve())) - - // Generate the Thumbnail, Vibrant, and SHA1 hash - .then(() => Promise.all([Thumbnail(req.file), Vibrant(req.file), Hash(req.file)])) - // skipcq: JS-0086 - .then(([thumbnail, vibrant, sha1]) => ( - req.file.thumbnail = thumbnail,// skipcq: JS-0090 - req.file.vibrant = vibrant,// skipcq: JS-0090 - req.file.sha1 = sha1// skipcq: JS-0090 - )) - - // Remove the temp file if using S3 storage, otherwise rename the local file - .then(() => (s3enabled ? fs.remove(path(diskFilePath, req.file.originalname)) : renameFile(req, saveAsOriginal ? req.file.originalname : req.file.sha1))) - .then(() => next()) - .catch((err) => next(err)); - }); + // Upload file + app.post('/', doUpload, processUploaded, ({ next }) => next()); // Process uploaded file app.post('/', (req, res) => { @@ -275,7 +241,7 @@ function startup() { }), local: () => { res.header('Accept-Ranges', 'bytes').header('Content-Length', fileData.size).type(fileData.mimetype); - fs.createReadStream(path(fileData.path)).pipe(res); + fs.createReadStream(fileData.path).pipe(res); } }; @@ -334,6 +300,11 @@ function startup() { .catch(console.error); }); + app.use((err, _req, res, _next) => { + console.error(err); + res.sendStatus(500); + }); + // Host the server app.listen(port, host, () => log(`Server started on [${host}:${port}]\nAuthorized users: ${Object.keys(users).length}\nAvailable files: ${Object.keys(data).length}`)); } diff --git a/hash.js b/hash.js index 47e0d84..c71529b 100644 --- a/hash.js +++ b/hash.js @@ -1,12 +1,15 @@ const fs = require('fs-extra'); const crypto = require('crypto'); const toArray = require('stream-to-array') -const { path } = require('./utils'); -const { s3enabled, diskFilePath } = require('./config.json'); +/** + * Generates a SHA1 hash for the provided file + * @param {*} file The file to hash + * @returns The SHA1 hash + */ module.exports = (file) => new Promise((resolve, reject) => - toArray((fs.createReadStream(s3enabled ? path(diskFilePath, file.originalname) : path(file.path)))) + toArray((fs.createReadStream(file.path))) .then((parts) => Buffer.concat(parts.map((part) => (Buffer.isBuffer(part) ? part : Buffer.from(part))))) .then((buf) => crypto.createHash('sha1').update(buf).digest('hex')) // skipcq: JS-D003 .then(resolve) diff --git a/package-lock.json b/package-lock.json index aaa1a92..e992361 100755 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,7 @@ "jimp": "^0.16.1", "luxon": "^1.26.0", "marked": "^2.0.7", - "multer": "^1.4.2", + "multer": "^2.0.0-rc.2", "node-fetch": "^2.6.1", "node-vibrant": "*", "prompt": "^1.1.0", @@ -836,6 +836,14 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base32-encode": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/base32-encode/-/base32-encode-1.2.0.tgz", + "integrity": "sha512-cHFU8XeRyx0GgmoWi5qHMCVRiqU6J3MHWxVgun7jggCBUpVzm1Ir7M9dYr2whjSNc3tFeXfQ/oZjQu/4u55h9A==", + "dependencies": { + "to-data-view": "^1.1.0" + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -913,38 +921,16 @@ "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" }, "node_modules/busboy": { - "version": "0.2.14", - "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz", - "integrity": "sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=", + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.3.1.tgz", + "integrity": "sha512-y7tTxhGKXcyBxRKAni+awqx8uqaJKrSFSNFSeRG5CsWNdmy2BIK+6VGWEW7TZnIO/533mtMEA4rOevQV815YJw==", "dependencies": { - "dicer": "0.2.5", - "readable-stream": "1.1.x" + "dicer": "0.3.0" }, "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/busboy/node_modules/isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" - }, - "node_modules/busboy/node_modules/readable-stream": { - "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", - "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.1", - "isarray": "0.0.1", - "string_decoder": "~0.10.x" + "node": ">=4.5.0" } }, - "node_modules/busboy/node_modules/string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" - }, "node_modules/bytes": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", @@ -1116,38 +1102,16 @@ "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }, "node_modules/dicer": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz", - "integrity": "sha1-WZbAhrszIYyBLAkL3cCc0S+stw8=", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.3.0.tgz", + "integrity": "sha512-MdceRRWqltEG2dZqO769g27N/3PXfcKl04VhYnBlo2YhH7zPi88VebsjTKclaOyiuMaGU72hTfw3VkUitGcVCA==", "dependencies": { - "readable-stream": "1.1.x", "streamsearch": "0.1.2" }, "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/dicer/node_modules/isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" - }, - "node_modules/dicer/node_modules/readable-stream": { - "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", - "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.1", - "isarray": "0.0.1", - "string_decoder": "~0.10.x" + "node": ">=4.5.0" } }, - "node_modules/dicer/node_modules/string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" - }, "node_modules/discord.js": { "version": "12.5.3", "resolved": "https://registry.npmjs.org/discord.js/-/discord.js-12.5.3.tgz", @@ -1181,6 +1145,11 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, + "node_modules/encode-utf8": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/encode-utf8/-/encode-utf8-1.0.3.tgz", + "integrity": "sha512-ucAnuBEhUK4boH2HjVYG5Q2mQyPorvv0u/ocS+zhdw0S8AlHYY+GOFhP1Gio5z4icpP2ivFSvhtFjQi8+T9ppw==" + }, "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -1189,6 +1158,14 @@ "node": ">= 0.8" } }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/env-paths": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", @@ -1336,6 +1313,14 @@ "node": ">= 0.8" } }, + "node_modules/fmix": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/fmix/-/fmix-0.1.0.tgz", + "integrity": "sha1-x7vxJN7ELJ0ZHPuUfQqXeN2YbAw=", + "dependencies": { + "imul": "^1.0.0" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -1366,6 +1351,14 @@ "node": ">=10" } }, + "node_modules/fs-temp": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/fs-temp/-/fs-temp-1.2.1.tgz", + "integrity": "sha512-okTwLB7/Qsq82G6iN5zZJFsOfZtx2/pqrA7Hk/9fvy+c+eJS9CvgGXT2uNxwnI14BDY9L/jQPkaBgSvlKfSW9w==", + "dependencies": { + "random-path": "^0.1.0" + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -1442,6 +1435,11 @@ "node": ">= 0.4.0" } }, + "node_modules/has-own-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-own-property/-/has-own-property-1.0.0.tgz", + "integrity": "sha1-PpjMHStfPagNPq/SjMZfyt2pZdo=" + }, "node_modules/has-symbols": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", @@ -1554,6 +1552,14 @@ "node": ">=0.9.0" } }, + "node_modules/imul": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/imul/-/imul-1.0.1.tgz", + "integrity": "sha1-nVhnFh6LPelsLDjV3HyxAvNeKsk=", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -1821,21 +1827,33 @@ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, "node_modules/multer": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.2.tgz", - "integrity": "sha512-xY8pX7V+ybyUpbYMxtjM9KAiD9ixtg5/JkeKUTD6xilfDv0vzzOFcCp4Ljb1UU3tSOM3VTZtKo63OmzOrGi3Cg==", + "version": "2.0.0-rc.2", + "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.0-rc.2.tgz", + "integrity": "sha512-IjQe1wZoANXrZ0A7cED1dxUny3BFezp6jajZ2FDjP6Rjfxib20WPbWYIPe2mjq0enipuIqz7XLdAkbqO6+oqzQ==", "dependencies": { "append-field": "^1.0.0", - "busboy": "^0.2.11", - "concat-stream": "^1.5.2", - "mkdirp": "^0.5.1", - "object-assign": "^4.1.1", + "busboy": "^0.3.1", + "bytes": "^3.1.0", + "fs-temp": "^1.1.1", + "has-own-property": "^1.0.0", "on-finished": "^2.3.0", - "type-is": "^1.6.4", - "xtend": "^4.0.0" + "pify": "^5.0.0", + "pump": "^3.0.0", + "stream-file-type": "^0.4.0", + "type-is": "^1.6.18" }, "engines": { - "node": ">= 0.10.0" + "node": ">=10.13" + } + }, + "node_modules/murmur-32": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/murmur-32/-/murmur-32-0.2.0.tgz", + "integrity": "sha512-ZkcWZudylwF+ir3Ld1n7gL6bI2mQAzXvSobPwVtu8aYi2sbXeipeSkdcanRLzIofLcM5F53lGaKm2dk7orBi7Q==", + "dependencies": { + "encode-utf8": "^1.0.3", + "fmix": "^0.1.0", + "imul": "^1.0.0" } }, "node_modules/mute-stream": { @@ -1999,6 +2017,17 @@ "resolved": "https://registry.npmjs.org/phin/-/phin-2.9.3.tgz", "integrity": "sha512-CzFr90qM24ju5f88quFC/6qohjC144rehe5n6DH900lgXmUe86+xCKc10ev56gRKC4/BkHUoG4uSiQgBiIXwDA==" }, + "node_modules/pify": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-5.0.0.tgz", + "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/pixelmatch": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-4.0.2.tgz", @@ -2211,6 +2240,15 @@ "resolved": "https://registry.npmjs.org/pug-walk/-/pug-walk-2.0.0.tgz", "integrity": "sha512-yYELe9Q5q9IQhuvqsZNwA5hfPkMJ8u92bQLIMcsMxf/VADjNtEYptU+inlufAFYcWdHlwNfZOEnOOQrZrcyJCQ==" }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -2251,6 +2289,15 @@ } ] }, + "node_modules/random-path": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/random-path/-/random-path-0.1.2.tgz", + "integrity": "sha512-4jY0yoEaQ5v9StCl5kZbNIQlg1QheIDBrdkDn53EynpPb9FgO6//p3X/tgMnrC45XN6QZCzU1Xz/+pSSsJBpRw==", + "dependencies": { + "base32-encode": "^0.1.0 || ^1.0.0", + "murmur-32": "^0.1.0 || ^0.2.0" + } + }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -2476,6 +2523,39 @@ "node": ">= 0.6" } }, + "node_modules/stream-file-type": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/stream-file-type/-/stream-file-type-0.4.0.tgz", + "integrity": "sha512-Nj1pls0XYICU1yWqH1Qi4le/IY8b8vMtW9z4Jk0G/Fd/5p851w/kLXW94Lbv/r4CZA2eXIj4/EHNTaPnUxT8Og==", + "dependencies": { + "file-type": "^12.0.0", + "readable-stream": "^3.0.6" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/stream-file-type/node_modules/file-type": { + "version": "12.4.2", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-12.4.2.tgz", + "integrity": "sha512-UssQP5ZgIOKelfsaB5CuGAL+Y+q7EmONuiwF3N5HAH0t27rvrttgi6Ra9k/+DVaY9UF6+ybxu5pOXLUdA8N7Vg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/stream-file-type/node_modules/readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/stream-to-array": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/stream-to-array/-/stream-to-array-2.3.0.tgz", @@ -2530,6 +2610,11 @@ "node": "*" } }, + "node_modules/to-data-view": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/to-data-view/-/to-data-view-1.1.0.tgz", + "integrity": "sha512-1eAdufMg6mwgmlojAx3QeMnzB/BTVp7Tbndi3U7ftcT2zCZadjxkkmLmd97zmaxWi+sgGcgWrokmpEoy0Dn0vQ==" + }, "node_modules/to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -3489,6 +3574,14 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "base32-encode": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/base32-encode/-/base32-encode-1.2.0.tgz", + "integrity": "sha512-cHFU8XeRyx0GgmoWi5qHMCVRiqU6J3MHWxVgun7jggCBUpVzm1Ir7M9dYr2whjSNc3tFeXfQ/oZjQu/4u55h9A==", + "requires": { + "to-data-view": "^1.1.0" + } + }, "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -3546,35 +3639,11 @@ "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" }, "busboy": { - "version": "0.2.14", - "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz", - "integrity": "sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=", + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.3.1.tgz", + "integrity": "sha512-y7tTxhGKXcyBxRKAni+awqx8uqaJKrSFSNFSeRG5CsWNdmy2BIK+6VGWEW7TZnIO/533mtMEA4rOevQV815YJw==", "requires": { - "dicer": "0.2.5", - "readable-stream": "1.1.x" - }, - "dependencies": { - "isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" - }, - "readable-stream": { - "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", - "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.1", - "isarray": "0.0.1", - "string_decoder": "~0.10.x" - } - }, - "string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" - } + "dicer": "0.3.0" } }, "bytes": { @@ -3712,35 +3781,11 @@ "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }, "dicer": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz", - "integrity": "sha1-WZbAhrszIYyBLAkL3cCc0S+stw8=", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.3.0.tgz", + "integrity": "sha512-MdceRRWqltEG2dZqO769g27N/3PXfcKl04VhYnBlo2YhH7zPi88VebsjTKclaOyiuMaGU72hTfw3VkUitGcVCA==", "requires": { - "readable-stream": "1.1.x", "streamsearch": "0.1.2" - }, - "dependencies": { - "isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" - }, - "readable-stream": { - "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", - "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.1", - "isarray": "0.0.1", - "string_decoder": "~0.10.x" - } - }, - "string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" - } } }, "discord.js": { @@ -3773,11 +3818,24 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, + "encode-utf8": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/encode-utf8/-/encode-utf8-1.0.3.tgz", + "integrity": "sha512-ucAnuBEhUK4boH2HjVYG5Q2mQyPorvv0u/ocS+zhdw0S8AlHYY+GOFhP1Gio5z4icpP2ivFSvhtFjQi8+T9ppw==" + }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" }, + "end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "requires": { + "once": "^1.4.0" + } + }, "env-paths": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", @@ -3890,6 +3948,14 @@ "unpipe": "~1.0.0" } }, + "fmix": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/fmix/-/fmix-0.1.0.tgz", + "integrity": "sha1-x7vxJN7ELJ0ZHPuUfQqXeN2YbAw=", + "requires": { + "imul": "^1.0.0" + } + }, "forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -3911,6 +3977,14 @@ "universalify": "^2.0.0" } }, + "fs-temp": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/fs-temp/-/fs-temp-1.2.1.tgz", + "integrity": "sha512-okTwLB7/Qsq82G6iN5zZJFsOfZtx2/pqrA7Hk/9fvy+c+eJS9CvgGXT2uNxwnI14BDY9L/jQPkaBgSvlKfSW9w==", + "requires": { + "random-path": "^0.1.0" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -3975,6 +4049,11 @@ "function-bind": "^1.1.1" } }, + "has-own-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-own-property/-/has-own-property-1.0.0.tgz", + "integrity": "sha1-PpjMHStfPagNPq/SjMZfyt2pZdo=" + }, "has-symbols": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", @@ -4059,6 +4138,11 @@ "resolved": "https://registry.npmjs.org/image-q/-/image-q-1.1.1.tgz", "integrity": "sha1-/IQJlmRGC5DKhi2TALa/u7+/gFY=" }, + "imul": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/imul/-/imul-1.0.1.tgz", + "integrity": "sha1-nVhnFh6LPelsLDjV3HyxAvNeKsk=" + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -4270,18 +4354,30 @@ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, "multer": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.2.tgz", - "integrity": "sha512-xY8pX7V+ybyUpbYMxtjM9KAiD9ixtg5/JkeKUTD6xilfDv0vzzOFcCp4Ljb1UU3tSOM3VTZtKo63OmzOrGi3Cg==", + "version": "2.0.0-rc.2", + "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.0-rc.2.tgz", + "integrity": "sha512-IjQe1wZoANXrZ0A7cED1dxUny3BFezp6jajZ2FDjP6Rjfxib20WPbWYIPe2mjq0enipuIqz7XLdAkbqO6+oqzQ==", "requires": { "append-field": "^1.0.0", - "busboy": "^0.2.11", - "concat-stream": "^1.5.2", - "mkdirp": "^0.5.1", - "object-assign": "^4.1.1", + "busboy": "^0.3.1", + "bytes": "^3.1.0", + "fs-temp": "^1.1.1", + "has-own-property": "^1.0.0", "on-finished": "^2.3.0", - "type-is": "^1.6.4", - "xtend": "^4.0.0" + "pify": "^5.0.0", + "pump": "^3.0.0", + "stream-file-type": "^0.4.0", + "type-is": "^1.6.18" + } + }, + "murmur-32": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/murmur-32/-/murmur-32-0.2.0.tgz", + "integrity": "sha512-ZkcWZudylwF+ir3Ld1n7gL6bI2mQAzXvSobPwVtu8aYi2sbXeipeSkdcanRLzIofLcM5F53lGaKm2dk7orBi7Q==", + "requires": { + "encode-utf8": "^1.0.3", + "fmix": "^0.1.0", + "imul": "^1.0.0" } }, "mute-stream": { @@ -4419,6 +4515,11 @@ "resolved": "https://registry.npmjs.org/phin/-/phin-2.9.3.tgz", "integrity": "sha512-CzFr90qM24ju5f88quFC/6qohjC144rehe5n6DH900lgXmUe86+xCKc10ev56gRKC4/BkHUoG4uSiQgBiIXwDA==" }, + "pify": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-5.0.0.tgz", + "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA==" + }, "pixelmatch": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-4.0.2.tgz", @@ -4594,6 +4695,15 @@ "resolved": "https://registry.npmjs.org/pug-walk/-/pug-walk-2.0.0.tgz", "integrity": "sha512-yYELe9Q5q9IQhuvqsZNwA5hfPkMJ8u92bQLIMcsMxf/VADjNtEYptU+inlufAFYcWdHlwNfZOEnOOQrZrcyJCQ==" }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -4614,6 +4724,15 @@ "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" }, + "random-path": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/random-path/-/random-path-0.1.2.tgz", + "integrity": "sha512-4jY0yoEaQ5v9StCl5kZbNIQlg1QheIDBrdkDn53EynpPb9FgO6//p3X/tgMnrC45XN6QZCzU1Xz/+pSSsJBpRw==", + "requires": { + "base32-encode": "^0.1.0 || ^1.0.0", + "murmur-32": "^0.1.0 || ^0.2.0" + } + }, "range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -4789,6 +4908,32 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, + "stream-file-type": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/stream-file-type/-/stream-file-type-0.4.0.tgz", + "integrity": "sha512-Nj1pls0XYICU1yWqH1Qi4le/IY8b8vMtW9z4Jk0G/Fd/5p851w/kLXW94Lbv/r4CZA2eXIj4/EHNTaPnUxT8Og==", + "requires": { + "file-type": "^12.0.0", + "readable-stream": "^3.0.6" + }, + "dependencies": { + "file-type": { + "version": "12.4.2", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-12.4.2.tgz", + "integrity": "sha512-UssQP5ZgIOKelfsaB5CuGAL+Y+q7EmONuiwF3N5HAH0t27rvrttgi6Ra9k/+DVaY9UF6+ybxu5pOXLUdA8N7Vg==" + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "stream-to-array": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/stream-to-array/-/stream-to-array-2.3.0.tgz", @@ -4830,6 +4975,11 @@ "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.4.2.tgz", "integrity": "sha512-vJhccZPs965sV/L2sU4oRQVAos0pQXwsvTLkWYdqJ+a8Q5kPFzJTuOFwy7UniPli44NKQGAglksjvOcpo95aZA==" }, + "to-data-view": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/to-data-view/-/to-data-view-1.1.0.tgz", + "integrity": "sha512-1eAdufMg6mwgmlojAx3QeMnzB/BTVp7Tbndi3U7ftcT2zCZadjxkkmLmd97zmaxWi+sgGcgWrokmpEoy0Dn0vQ==" + }, "to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", diff --git a/package.json b/package.json index bd4a0ec..03c6d61 100755 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "jimp": "^0.16.1", "luxon": "^1.26.0", "marked": "^2.0.7", - "multer": "^1.4.2", + "multer": "^2.0.0-rc.2", "node-fetch": "^2.6.1", "node-vibrant": "*", "prompt": "^1.1.0", @@ -48,4 +48,4 @@ "tycrek-s3-transform": "^3.3.0", "uuid": "^8.3.2" } -} \ No newline at end of file +} diff --git a/storage.js b/storage.js index 558ea4c..78c619c 100644 --- a/storage.js +++ b/storage.js @@ -4,45 +4,90 @@ const fs = require('fs-extra'); const aws = require('aws-sdk'); const multer = require('multer'); -const multerS3 = require('tycrek-s3-transform'); -const { getSafeExt } = require('./utils'); -const { diskFilePath, saveWithDate, s3enabled, s3endpoint, s3bucket, s3accessKey, s3secretKey } = require('./config.json'); +const Thumbnail = require('./thumbnails'); +const Vibrant = require('./vibrant'); +const Hash = require('./hash'); +const { getSafeExt, getDatedDirname, sanitize, generateId } = require('./utils'); +const { s3enabled, s3endpoint, s3bucket, s3accessKey, s3secretKey, saveAsOriginal } = require('./config.json'); const s3 = new aws.S3({ endpoint: new aws.Endpoint(s3endpoint), credentials: new aws.Credentials({ accessKeyId: s3accessKey, secretAccessKey: s3secretKey }) }); -const uploadS3 = multer({ - storage: multerS3({ - s3, - bucket: s3bucket, - acl: 'public-read', - key: (req, file, cb) => cb(null, req.randomId.concat(getSafeExt(file.mimetype))), - contentType: (_req, file, cb) => cb(null, file.mimetype) - }) -}).single('file'); - -const deleteS3 = (file) => - new Promise((resolve, reject) => - s3.deleteObject({ Bucket: s3bucket, Key: file.randomId.concat(getSafeExt(file.mimetype)) }).promise().then(resolve).catch(reject)); - -const uploadLocal = multer({ - storage: multer.diskStorage({ - destination: !saveWithDate ? diskFilePath : (_req, _file, cb) => { - // Get current month and year - const [month, , year] = new Date().toLocaleDateString('en-US').split('/'); - - // Add 0 before single digit months (6 turns into 06) - const folder = `${diskFilePath}/${year}-${`0${month}`.slice(-2)}`; // skipcq: JS-0074 - - // Create folder if it doesn't exist - fs.ensureDirSync(folder); - - cb(null, folder); - } - }) -}).single('file'); +function saveFile(req) { + return new Promise((resolve, reject) => + fs.ensureDir(getDatedDirname()) + .then(() => fs.createWriteStream(req.file.path.concat('.temp'))) + .then((stream) => req.file.stream.pipe(stream).on('finish', resolve).on('error', reject)) + .catch(reject)); +} + +function getLocalFilename(req) { + return `${getDatedDirname()}/${saveAsOriginal ? req.file.originalname : req.file.sha1}`; +} + +function processUploaded(req, _, next) { + // Fixes + req.file.mimetype = req.file.detectedMimeType; + req.file.originalname = sanitize(req.file.originalName); + req.file.randomId = generateId('random', 32, null, null); + req.file.deleteId = generateId('random', 32, null, null); + + // Remove unwanted fields + delete req.file.fieldName; + delete req.file.originalName; + delete req.file.clientReportedMimeType; + delete req.file.clientReportedFileExtension; + delete req.file.detectedMimeType; + delete req.file.detectedFileExtension; + + // Operations + saveFile(req) + .then(() => req.file.path = req.file.path.concat('.temp')) + .then(() => Promise.all([Thumbnail(req.file), Vibrant(req.file), Hash(req.file)])) + .then(([thumbnail, vibrant, sha1]) => ( + req.file.thumbnail = thumbnail, // skipcq: JS-0090 + req.file.vibrant = vibrant, // skipcq: JS-0090 + req.file.sha1 = sha1 // skipcq: JS-0090 + )) + + .then(() => + new Promise((resolve, reject) => s3enabled + + // Upload to Amazon S3 + ? s3.putObject({ + Bucket: s3bucket, + Key: req.file.randomId.concat(getSafeExt(req.file.mimetype)), + ACL: 'public-read', + ContentType: req.file.mimetype, + Body: fs.createReadStream(req.file.path) + }).promise().then(resolve).catch(reject) + + // Save to local storage + : fs.ensureDir(getDatedDirname()) + .then(() => fs.copy(req.file.path, getLocalFilename(req), { preserveTimestamps: true })) + .then(resolve) + .catch(reject) + )) + .then(() => fs.remove(req.file.path)) + .then(() => !s3enabled && (req.file.path = getLocalFilename(req))) // skipcq: JS-0090 + .then(() => delete req.file.stream) + .then(() => next()) + .catch(next); +} + +function deleteS3(file) { + return new Promise((resolve, reject) => s3 + .deleteObject({ Bucket: s3bucket, Key: file.randomId.concat(getSafeExt(file.mimetype)) }) + .promise() + .then(resolve) + .catch(reject)); +} + +function bucketSize() { + return new Promise((resolve, reject) => (s3enabled ? listAllKeys(resolve, reject) : resolve(0))); +} function listAllKeys(resolve, reject, token) { let allKeys = []; @@ -51,7 +96,9 @@ function listAllKeys(resolve, reject, token) { .catch(reject); } -const bucketSize = () => - new Promise((resolve, reject) => (s3enabled ? listAllKeys(resolve, reject) : resolve(0))); - -module.exports = { uploadLocal, uploadS3, deleteS3, bucketSize }; +module.exports = { + doUpload: multer({ limits: { fileSize: '100MB' } }).single('file'), + processUploaded, + deleteS3, + bucketSize +}; diff --git a/thumbnails.js b/thumbnails.js index 553bced..44f2d91 100644 --- a/thumbnails.js +++ b/thumbnails.js @@ -2,38 +2,67 @@ const ffmpeg = require('ffmpeg-static'); const Jimp = require('jimp'); const shell = require('any-shell-escape'); const { exec } = require('child_process'); -const { path, getS3url } = require('./utils'); -const { s3enabled, diskFilePath } = require('./config.json'); +const { path } = require('./utils'); +const { diskFilePath } = require('./config.json'); +// Thumbnail parameters const THUMBNAIL_QUALITY = 50; const THUMBNAIL_SIZE = 512; +/** + * Builds a safe escaped ffmpeg command + * @param {String} src Path to the input file + * @param {String} dest Path of the output file + * @returns {String} The command to execute + */ function getCommand(src, dest) { return shell([ - ffmpeg, '-y', '-v', process.env.NODE_ENV === 'production' ? 'error' : 'debug', - '-i', src, - '-ss', '00:00:01.000', - '-frames:v', '1', - '-s', `${THUMBNAIL_SIZE}x${THUMBNAIL_SIZE}`, - dest + ffmpeg, '-y', + '-v', (process.env.NODE_ENV === 'production' ? 'error' : 'debug'), // Log level + '-i', src, // Input file + '-ss', '00:00:01.000', // Timestamp of frame to grab + '-frames:v', '1', // Number of frames to grab + '-s', `${THUMBNAIL_SIZE}x${THUMBNAIL_SIZE}`, // Dimensions of output file + dest // Output file ]); } +/** + * Builds a thumbnail filename + * @param {String} oldName The original filename + * @returns {String} The filename for the thumbnail + */ function getNewName(oldName) { return oldName.concat('.thumbnail.jpg'); } +/** + * Builds a path to the thumbnails + * @param {String} oldName The original filename + * @returns {String} The path to the thumbnail + */ function getNewNamePath(oldName) { return path(diskFilePath, 'thumbnails/', getNewName(oldName)); } +/** + * Extracts an image from a video file to use as a thumbnail, using ffmpeg + * @param {*} file The video file to pull a frame from + */ function getVideoThumbnail(file) { - return new Promise((resolve, reject) => exec(getCommand((s3enabled ? path(diskFilePath, file.originalname) : path(file.path)), getNewNamePath(file.originalname)), (err) => (err ? reject(err) : resolve()))); + return new Promise((resolve, reject) => exec( + getCommand(file.path, getNewNamePath(file.originalname)), + (err) => (err ? reject(err) : resolve()) + )); } -function getResizedThumbnail(file) { +/** + * Generates a thumbnail for the provided image + * @param {*} file The file to generate a thumbnail for + */ +function getImageThumbnail(file) { return new Promise((resolve, reject) => - Jimp.read(s3enabled ? getS3url(file.randomId, file.mimetype) : path(file.path)) + Jimp.read(file.path) .then((image) => image .quality(THUMBNAIL_QUALITY) .resize(THUMBNAIL_SIZE, THUMBNAIL_SIZE, Jimp.RESIZE_BICUBIC) @@ -42,8 +71,13 @@ function getResizedThumbnail(file) { .catch(reject)); } +/** + * Generates a thumbnail + * @param {*} file The file to generate a thumbnail for + * @returns The thumbnail filename (NOT the path) + */ module.exports = (file) => new Promise((resolve, reject) => - (file.mimetype.includes('video') ? getVideoThumbnail : getResizedThumbnail)(file) + (file.mimetype.includes('video') ? getVideoThumbnail : getImageThumbnail)(file) .then(() => resolve(getNewName(file.originalname))) .catch(reject)); diff --git a/utils.js b/utils.js index b601ca4..df2dadd 100755 --- a/utils.js +++ b/utils.js @@ -7,7 +7,7 @@ const token = require('./generators/token'); const zwsGen = require('./generators/zws'); const randomGen = require('./generators/random'); const gfyGen = require('./generators/gfycat'); -const { useSsl, port, domain, isProxied, diskFilePath, s3bucket, s3endpoint } = require('./config.json'); +const { useSsl, port, domain, isProxied, diskFilePath, saveWithDate, s3bucket, s3endpoint } = require('./config.json'); const { HTTP, HTTPS, KILOBYTES } = require('./MagicNumbers.json'); const path = (...paths) => Path.join(__dirname, ...paths); @@ -62,6 +62,16 @@ function replaceholder(data, { size, timestamp, originalname }) { .replace(/×tamp/g, formatTimestamp(timestamp)); } +function getDatedDirname() { + if (!saveWithDate) return diskFilePath; + + // Get current month and year + const [month, , year] = new Date().toLocaleDateString('en-US').split('/'); + + // Add 0 before single digit months (6 turns into 06) + return `${diskFilePath}${diskFilePath.endsWith('/') ? '' : '/'}${year}-${`0${month}`.slice(-2)}`; // skipcq: JS-0074 +} + const idModes = { zws: 'zws', // Zero-width spaces (see: https://zws.im/) og: 'original', // Use original uploaded filename @@ -84,6 +94,7 @@ module.exports = { formatTimestamp, formatBytes, replaceholder, + getDatedDirname, getSafeExt, randomHexColour, sanitize, diff --git a/vibrant.js b/vibrant.js index 9d4ee45..8290c16 100644 --- a/vibrant.js +++ b/vibrant.js @@ -1,15 +1,28 @@ const Vibrant = require('node-vibrant'); -const { path, randomHexColour } = require('./utils'); -const { s3enabled, diskFilePath } = require('./config.json'); +const { randomHexColour } = require('./utils'); +// Vibrant parameters const COLOR_COUNT = 256; const QUALITY = 3; -module.exports = (file) => - new Promise((resolve, reject) => ( - file.mimetype.includes('video') - ? resolve(randomHexColour()) - : Vibrant.from(s3enabled ? path(diskFilePath, file.originalname) : path(file.path)) - .maxColorCount(COLOR_COUNT).quality(QUALITY).getPalette() - .then((palettes) => resolve(palettes[Object.keys(palettes).sort((a, b) => palettes[b].population - palettes[a].population)[0]].hex)) - .catch(reject))); +/** + * Extracts a prominent colour from the provided image file + * @param {*} file The image to extract a colour from + * @param {*} resolve Runs if Promise was successful + * @param {*} reject Runs if Promise failed + */ +function getVibrant(file, resolve, reject) { + Vibrant.from(file.path) + .maxColorCount(COLOR_COUNT) + .quality(QUALITY) + .getPalette() + .then((palettes) => resolve(palettes[Object.keys(palettes).sort((a, b) => palettes[b].population - palettes[a].population)[0]].hex)) + .catch(reject); +} + +/** + * Extracts a colour from an image file. Returns a random Hex value if provided file is a video + * @param {*} file The file to get a colour from + * @returns The Vibrant colour as a Hex value (or random Hex value for videos) + */ +module.exports = (file) => new Promise((resolve, reject) => file.mimetype.includes('video') ? resolve(randomHexColour()) : getVibrant(file, resolve, reject)); From 8add5f7ecd0099748f53cc3bcf16fee4ed31a07d Mon Sep 17 00:00:00 2001 From: tycrek Date: Tue, 29 Jun 2021 18:07:22 -0600 Subject: [PATCH 2/7] fixed 500 instead of 401 when auth header missing --- ass.js | 1 + 1 file changed, 1 insertion(+) diff --git a/ass.js b/ass.js index c6ac6c3..5140c85 100755 --- a/ass.js +++ b/ass.js @@ -106,6 +106,7 @@ function startup() { // Block unauthorized requests and attempt token sanitization app.post('/', (req, res, next) => { + req.headers.authorization = req.headers.authorization || ''; req.token = req.headers.authorization.replace(/[^\da-z]/gi, ''); // Strip anything that isn't a digit or ASCII letter !verify(req, users) ? res.sendStatus(CODE_UNAUTHORIZED) : next(); // skipcq: JS-0093 }); From 0919939ac71485e3251999c423949ed63422b3de Mon Sep 17 00:00:00 2001 From: tycrek Date: Tue, 29 Jun 2021 18:09:04 -0600 Subject: [PATCH 3/7] added configurable upload limits & proper upload limit handling (need to re-run setup) --- ass.js | 3 ++- setup.js | 7 +++++++ storage.js | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ass.js b/ass.js index 5140c85..a576ac6 100755 --- a/ass.js +++ b/ass.js @@ -7,7 +7,7 @@ try { } // Load the config -const { host, port, useSsl, resourceIdSize, diskFilePath, gfyIdSize, resourceIdType, isProxied, s3enabled } = require('./config.json'); +const { host, port, maxUploadSize, useSsl, resourceIdSize, diskFilePath, gfyIdSize, resourceIdType, isProxied, s3enabled } = require('./config.json'); //#region Imports const fs = require('fs-extra'); @@ -113,6 +113,7 @@ function startup() { // Upload file app.post('/', doUpload, processUploaded, ({ next }) => next()); + app.use('/', (err, _req, res, next) => err.code && err.code === 'LIMIT_FILE_SIZE' ? res.status(413).send(`Max upload size: ${maxUploadSize}MB`) : next(err)); // Process uploaded file app.post('/', (req, res) => { diff --git a/setup.js b/setup.js index a72091f..30a170a 100755 --- a/setup.js +++ b/setup.js @@ -3,6 +3,7 @@ const config = { host: '0.0.0.0', port: 40115, domain: 'upload.example.com', + maxUploadSize: 50, useSsl: true, isProxied: true, resourceIdSize: 12, @@ -55,6 +56,12 @@ if (require.main === module) { required: true, message: 'You must input a valid domain name or IP to continue' }, + maxUploadSize: { + description: `Max allowable uploaded filesize, in megabytes`, + type: 'integer', + default: config.maxUploadSize, + require: false + }, useSsl: { description: 'Use SSL (requires reverse proxy!)', type: 'boolean', diff --git a/storage.js b/storage.js index 78c619c..2ec51f5 100644 --- a/storage.js +++ b/storage.js @@ -8,7 +8,7 @@ const Thumbnail = require('./thumbnails'); const Vibrant = require('./vibrant'); const Hash = require('./hash'); const { getSafeExt, getDatedDirname, sanitize, generateId } = require('./utils'); -const { s3enabled, s3endpoint, s3bucket, s3accessKey, s3secretKey, saveAsOriginal } = require('./config.json'); +const { s3enabled, s3endpoint, s3bucket, s3accessKey, s3secretKey, saveAsOriginal, maxUploadSize } = require('./config.json'); const s3 = new aws.S3({ endpoint: new aws.Endpoint(s3endpoint), @@ -97,7 +97,7 @@ function listAllKeys(resolve, reject, token) { } module.exports = { - doUpload: multer({ limits: { fileSize: '100MB' } }).single('file'), + doUpload: multer({ limits: { fileSize: `${maxUploadSize}MB` } }).single('file'), processUploaded, deleteS3, bucketSize From 7d47c173e9fb51928c5aa162ef2e589f2f1766ea Mon Sep 17 00:00:00 2001 From: tycrek Date: Tue, 29 Jun 2021 18:23:19 -0600 Subject: [PATCH 4/7] DeepSource fixes --- MagicNumbers.json | 2 ++ ass.js | 8 ++++---- storage.js | 18 +++++++++++------- vibrant.js | 2 +- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/MagicNumbers.json b/MagicNumbers.json index 7a40859..bdecbfb 100644 --- a/MagicNumbers.json +++ b/MagicNumbers.json @@ -5,5 +5,7 @@ "CODE_BAD_REQUEST": 400, "CODE_UNAUTHORIZED": 401, "CODE_NOT_FOUND": 404, + "CODE_PAYLOAD_TOO_LARGE": 413, + "CODE_INTERNAL_SERVER_ERROR": 500, "KILOBYTES": 1024 } \ No newline at end of file diff --git a/ass.js b/ass.js index a576ac6..04025a9 100755 --- a/ass.js +++ b/ass.js @@ -22,7 +22,7 @@ const { WebhookClient, MessageEmbed } = require('discord.js'); const { doUpload, deleteS3, processUploaded } = require('./storage'); const { path, saveData, log, verify, getTrueHttp, getTrueDomain, generateToken, generateId, formatBytes, formatTimestamp, arrayEquals, getS3url, getDirectUrl, getSafeExt, getResourceColor, replaceholder } = require('./utils'); -const { CODE_NO_CONTENT, CODE_BAD_REQUEST, CODE_UNAUTHORIZED, CODE_NOT_FOUND } = require('./MagicNumbers.json'); +const { CODE_NO_CONTENT, CODE_BAD_REQUEST, CODE_UNAUTHORIZED, CODE_NOT_FOUND, CODE_PAYLOAD_TOO_LARGE, CODE_INTERNAL_SERVER_ERROR } = require('./MagicNumbers.json'); //#endregion //#region Variables, module setup @@ -113,7 +113,7 @@ function startup() { // Upload file app.post('/', doUpload, processUploaded, ({ next }) => next()); - app.use('/', (err, _req, res, next) => err.code && err.code === 'LIMIT_FILE_SIZE' ? res.status(413).send(`Max upload size: ${maxUploadSize}MB`) : next(err)); + app.use('/', (err, _req, res, next) => err.code && err.code === 'LIMIT_FILE_SIZE' ? res.status(CODE_PAYLOAD_TOO_LARGE).send(`Max upload size: ${maxUploadSize}MB`) : next(err)); // Process uploaded file app.post('/', (req, res) => { @@ -302,9 +302,9 @@ function startup() { .catch(console.error); }); - app.use((err, _req, res, _next) => { + app.use(([err, , res,]) => { console.error(err); - res.sendStatus(500); + res.sendStatus(CODE_INTERNAL_SERVER_ERROR); }); // Host the server diff --git a/storage.js b/storage.js index 2ec51f5..0d5032d 100644 --- a/storage.js +++ b/storage.js @@ -10,6 +10,8 @@ const Hash = require('./hash'); const { getSafeExt, getDatedDirname, sanitize, generateId } = require('./utils'); const { s3enabled, s3endpoint, s3bucket, s3accessKey, s3secretKey, saveAsOriginal, maxUploadSize } = require('./config.json'); +const ID_GEN_LENGTH = 32; + const s3 = new aws.S3({ endpoint: new aws.Endpoint(s3endpoint), credentials: new aws.Credentials({ accessKeyId: s3accessKey, secretAccessKey: s3secretKey }) @@ -31,8 +33,8 @@ function processUploaded(req, _, next) { // Fixes req.file.mimetype = req.file.detectedMimeType; req.file.originalname = sanitize(req.file.originalName); - req.file.randomId = generateId('random', 32, null, null); - req.file.deleteId = generateId('random', 32, null, null); + req.file.randomId = generateId('random', ID_GEN_LENGTH, null, null); + req.file.deleteId = generateId('random', ID_GEN_LENGTH, null, null); // Remove unwanted fields delete req.file.fieldName; @@ -44,8 +46,9 @@ function processUploaded(req, _, next) { // Operations saveFile(req) - .then(() => req.file.path = req.file.path.concat('.temp')) + .then(() => req.file.path = req.file.path.concat('.temp')) // skipcq: JS-0086 .then(() => Promise.all([Thumbnail(req.file), Vibrant(req.file), Hash(req.file)])) + // skipcq: JS-0086 .then(([thumbnail, vibrant, sha1]) => ( req.file.thumbnail = thumbnail, // skipcq: JS-0090 req.file.vibrant = vibrant, // skipcq: JS-0090 @@ -53,6 +56,7 @@ function processUploaded(req, _, next) { )) .then(() => + // skipcq: JS-0229 new Promise((resolve, reject) => s3enabled // Upload to Amazon S3 @@ -85,10 +89,6 @@ function deleteS3(file) { .catch(reject)); } -function bucketSize() { - return new Promise((resolve, reject) => (s3enabled ? listAllKeys(resolve, reject) : resolve(0))); -} - function listAllKeys(resolve, reject, token) { let allKeys = []; s3.listObjectsV2({ Bucket: s3bucket, ContinuationToken: token }).promise() @@ -96,6 +96,10 @@ function listAllKeys(resolve, reject, token) { .catch(reject); } +function bucketSize() { + return new Promise((resolve, reject) => (s3enabled ? listAllKeys(resolve, reject) : resolve(0))); +} + module.exports = { doUpload: multer({ limits: { fileSize: `${maxUploadSize}MB` } }).single('file'), processUploaded, diff --git a/vibrant.js b/vibrant.js index 8290c16..3165b82 100644 --- a/vibrant.js +++ b/vibrant.js @@ -25,4 +25,4 @@ function getVibrant(file, resolve, reject) { * @param {*} file The file to get a colour from * @returns The Vibrant colour as a Hex value (or random Hex value for videos) */ -module.exports = (file) => new Promise((resolve, reject) => file.mimetype.includes('video') ? resolve(randomHexColour()) : getVibrant(file, resolve, reject)); +module.exports = (file) => new Promise((resolve, reject) => file.mimetype.includes('video') ? resolve(randomHexColour()) : getVibrant(file, resolve, reject)); // skipcq: JS-0229 From a916a1676f90e03386ccd8348fe62355c7ee3079 Mon Sep 17 00:00:00 2001 From: tycrek Date: Tue, 29 Jun 2021 18:24:54 -0600 Subject: [PATCH 5/7] yes another DS fix --- ass.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ass.js b/ass.js index 04025a9..ba18378 100755 --- a/ass.js +++ b/ass.js @@ -113,7 +113,7 @@ function startup() { // Upload file app.post('/', doUpload, processUploaded, ({ next }) => next()); - app.use('/', (err, _req, res, next) => err.code && err.code === 'LIMIT_FILE_SIZE' ? res.status(CODE_PAYLOAD_TOO_LARGE).send(`Max upload size: ${maxUploadSize}MB`) : next(err)); + app.use('/', (err, _req, res, next) => err.code && err.code === 'LIMIT_FILE_SIZE' ? res.status(CODE_PAYLOAD_TOO_LARGE).send(`Max upload size: ${maxUploadSize}MB`) : next(err)); // skipcq: JS-0229 // Process uploaded file app.post('/', (req, res) => { From 994b9b3236efd77fa053d3ef51f4953f5c1e722e Mon Sep 17 00:00:00 2001 From: tycrek Date: Tue, 29 Jun 2021 18:29:23 -0600 Subject: [PATCH 6/7] Updated README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7c378de..e5883c9 100755 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ - ✔️ Usage metrics - ✔️ Thumbnail support - ✔️ Basic multi-user support +- ✔️ Configurable global upload limit (per-user coming soon!) - ✔️ Local storage *or* block-storage support for [Amazon S3](https://aws.amazon.com/s3/) (including [DigitalOcean Spaces](https://www.digitalocean.com/products/spaces/)) - ✔️ Multiple access types - **[ZWS](https://zws.im)** @@ -89,6 +90,7 @@ In your Cloudflare DNS dashboard, make sure your domain/subdomain is set to **DN - URL: `$json:.resource$` - Thumbnail: `$json:.thumbnail$` - Deletion URL: `$json:.delete$` + - Error message: `$response$` - MagicCap users: **do not** include the `.` in the above (i.e. `$json:resource$`) 6. The file `sample_config.sxcu` can also be modified & imported to suit your needs From c843f6d8579f50a44203bdb2385ae906d0c0727a Mon Sep 17 00:00:00 2001 From: tycrek Date: Tue, 29 Jun 2021 18:33:06 -0600 Subject: [PATCH 7/7] version bump --> 0.5.1 --- package-lock.json | 5 +++-- package.json | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index e992361..11f837f 100755 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ass", - "version": "0.5.0", + "version": "0.5.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "ass", - "version": "0.5.0", + "version": "0.5.1", "license": "ISC", "dependencies": { "any-shell-escape": "^0.1.1", @@ -2266,6 +2266,7 @@ "version": "0.2.0", "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "deprecated": "The", "engines": { "node": ">=0.4.x" } diff --git a/package.json b/package.json index 03c6d61..a03ec2f 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ass", - "version": "0.5.0", + "version": "0.5.1", "description": "The superior self-hosted ShareX server", "main": "ass.js", "scripts": { @@ -48,4 +48,4 @@ "tycrek-s3-transform": "^3.3.0", "uuid": "^8.3.2" } -} +} \ No newline at end of file