From 49668346096bdacd8e764ce028eb30b3bf940b66 Mon Sep 17 00:00:00 2001 From: tycrek Date: Tue, 6 Jul 2021 13:55:49 -0600 Subject: [PATCH] migrated to new StorageEngine system --- MagicNumbers.json | 1 - ass-x | 2 +- ass.js | 2 +- data.js | 12 +----- package-lock.json | 50 ++++++++++++++++++++++ package.json | 3 +- routers/resource.js | 102 ++++++++++++++++++++------------------------ routers/upload.js | 99 +++++++++++++++++++++--------------------- utils.js | 3 +- 9 files changed, 153 insertions(+), 121 deletions(-) diff --git a/MagicNumbers.json b/MagicNumbers.json index 074de1b..842c5e0 100644 --- a/MagicNumbers.json +++ b/MagicNumbers.json @@ -3,7 +3,6 @@ "HTTPS": 443, "CODE_OK": 200, "CODE_NO_CONTENT": 204, - "CODE_BAD_REQUEST": 400, "CODE_UNAUTHORIZED": 401, "CODE_NOT_FOUND": 404, "CODE_PAYLOAD_TOO_LARGE": 413, diff --git a/ass-x b/ass-x index 43c8082..7e27c22 160000 --- a/ass-x +++ b/ass-x @@ -1 +1 @@ -Subproject commit 43c8082e78d01d26f2e6c944e73bca67bb1d5197 +Subproject commit 7e27c22ce5ac86e19789d7f94e85ad4225ea0b0a diff --git a/ass.js b/ass.js index 70d95c5..72518cc 100755 --- a/ass.js +++ b/ass.js @@ -71,4 +71,4 @@ app.use(([err, , res,]) => { }); // Host the server -app.listen(port, host, () => log(`Server started on [${host}:${port}]\nAuthorized users: ${Object.keys(users).length}\nAvailable files: ${Object.keys(data).length}`)); +app.listen(port, host, () => log(`Server started on [${host}:${port}]\nAuthorized users: ${Object.keys(users).length}\nAvailable files: ${data.size}`)); diff --git a/data.js b/data.js index c128362..e28d932 100644 --- a/data.js +++ b/data.js @@ -2,14 +2,6 @@ * Used for global data management */ -const fs = require('fs-extra'); -const { log, path } = require('./utils'); - -// Make sure data.json exists -if (!fs.existsSync(path('data.json'))) { - fs.writeJsonSync(path('data.json'), {}, { spaces: 4 }); - log('File [data.json] created'); -} else log('File [data.json] exists'); - -const data = require('./data.json'); +const { JsonStorageEngine } = require('@tycrek/ass-storage-engine'); +const data = new JsonStorageEngine(); module.exports = data; diff --git a/package-lock.json b/package-lock.json index 2b2ec62..4acd3a4 100755 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.6.0", "license": "ISC", "dependencies": { + "@tycrek/ass-storage-engine": "0.2.4", "any-shell-escape": "^0.1.1", "aws-sdk": "^2.930.0", "check-node-version": "^4.1.0", @@ -560,6 +561,35 @@ "regenerator-runtime": "^0.13.3" } }, + "node_modules/@tycrek/ass-storage-engine": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@tycrek/ass-storage-engine/-/ass-storage-engine-0.2.4.tgz", + "integrity": "sha512-2LnfS1MQPKdB9Zb4e4oQcB+Pw8Z0+Q/mjoKowAH767WTxP3s1jD8yPFpTFOSRAoz52lrHJBMAEK31iIxqPBL/w==", + "dependencies": { + "fs-extra": "^10.0.0" + }, + "engines": { + "node": "^14.x.x", + "npm": "^7.x.x" + }, + "funding": { + "type": "patreon", + "url": "https://patreon.com/tycrek" + } + }, + "node_modules/@tycrek/ass-storage-engine/node_modules/fs-extra": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz", + "integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/@types/node": { "version": "10.17.60", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", @@ -3754,6 +3784,26 @@ "regenerator-runtime": "^0.13.3" } }, + "@tycrek/ass-storage-engine": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@tycrek/ass-storage-engine/-/ass-storage-engine-0.2.4.tgz", + "integrity": "sha512-2LnfS1MQPKdB9Zb4e4oQcB+Pw8Z0+Q/mjoKowAH767WTxP3s1jD8yPFpTFOSRAoz52lrHJBMAEK31iIxqPBL/w==", + "requires": { + "fs-extra": "^10.0.0" + }, + "dependencies": { + "fs-extra": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz", + "integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==", + "requires": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + } + } + }, "@types/node": { "version": "10.17.60", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", diff --git a/package.json b/package.json index dd0c6df..e5393c0 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ass", - "version": "0.6.0", + "version": "0.7.0", "description": "The superior self-hosted ShareX server", "main": "ass.js", "engines": { @@ -34,6 +34,7 @@ "url": "https://patreon.com/tycrek" }, "dependencies": { + "@tycrek/ass-storage-engine": "0.2.4", "any-shell-escape": "^0.1.1", "aws-sdk": "^2.930.0", "check-node-version": "^4.1.0", diff --git a/routers/resource.js b/routers/resource.js index c4a24ec..f1b53b4 100644 --- a/routers/resource.js +++ b/routers/resource.js @@ -3,8 +3,8 @@ const escape = require('escape-html'); const fetch = require('node-fetch'); const { deleteS3 } = require('../storage'); const { diskFilePath, s3enabled } = require('../config.json'); -const { path, saveData, log, getTrueHttp, getTrueDomain, formatBytes, formatTimestamp, getS3url, getDirectUrl, getSafeExt, getResourceColor, replaceholder } = require('../utils'); -const { CODE_BAD_REQUEST, CODE_UNAUTHORIZED, CODE_NOT_FOUND, } = require('../MagicNumbers.json'); +const { path, log, getTrueHttp, getTrueDomain, formatBytes, formatTimestamp, getS3url, getDirectUrl, getSafeExt, getResourceColor, replaceholder } = require('../utils'); +const { CODE_UNAUTHORIZED, CODE_NOT_FOUND, } = require('../MagicNumbers.json'); const data = require('../data'); const users = require('../auth'); @@ -14,16 +14,17 @@ const router = express.Router(); // Middleware for parsing the resource ID and handling 404 router.use((req, res, next) => { // Parse the resource ID - req.ass = { resourceId: escape(req.resourceId).split('.')[0] }; + req.ass = { resourceId: escape(req.resourceId || '').split('.')[0] }; - // If the ID is invalid, return 404. Otherwise, continue normally // skipcq: JS-0093 - (!req.ass.resourceId || !data[req.ass.resourceId]) ? res.sendStatus(CODE_NOT_FOUND) : next(); + // If the ID is invalid, return 404. Otherwise, continue normally + data.has(req.ass.resourceId) + .then((has) => has ? next() : res.sendStatus(CODE_NOT_FOUND)) + .catch(next); }); // View file -router.get('/', (req, res) => { +router.get('/', (req, res, next) => data.get(req.ass.resourceId).then((fileData) => { const { resourceId } = req.ass; - const fileData = data[resourceId]; const isVideo = fileData.mimetype.includes('video'); // Build OpenGraph meta tags @@ -47,15 +48,12 @@ router.get('/', (req, res) => { oembedUrl: `${getTrueHttp()}${getTrueDomain()}/${resourceId}/oembed`, ogtype: isVideo ? 'video.other' : 'image', urlType: `og:${isVideo ? 'video' : 'image'}`, - opengraph: replaceholder(ogs.join('\n'), fileData) + opengraph: replaceholder(ogs.join('\n'), fileData.size, fileData.timestamp, fileData.originalname) }); -}); +}).catch(next)); // Direct resource -router.get('/direct*', (req, res) => { - const { resourceId } = req.ass; - const fileData = data[resourceId]; - +router.get('/direct*', (req, res, next) => data.get(req.ass.resourceId).then((fileData) => { // Send file as an attachement for downloads if (req.query.download) res.header('Content-Disposition', `attachment; filename="${fileData.originalname}"`); @@ -73,58 +71,52 @@ router.get('/direct*', (req, res) => { }; uploaders[s3enabled ? 's3' : 'local'](); -}); +}).catch(next)); // Thumbnail response -router.get('/thumbnail', (req, res) => { - const { resourceId } = req.ass; - - // Read the file and send it to the client - fs.readFile(path(diskFilePath, 'thumbnails/', data[resourceId].thumbnail)) +router.get('/thumbnail', (req, res, next) => + data.get(req.ass.resourceId) + .then(({ thumbnail }) => fs.readFile(path(diskFilePath, 'thumbnails/', thumbnail))) .then((fileData) => res.type('jpg').send(fileData)) - .catch(console.error); -}); + .catch(next)); // oEmbed response for clickable authors/providers // https://oembed.com/ // https://old.reddit.com/r/discordapp/comments/82p8i6/a_basic_tutorial_on_how_to_get_the_most_out_of/ -router.get('/oembed', (req, res) => { - const { resourceId } = req.ass; - - // Build the oEmbed object & send the response - const { opengraph, mimetype } = data[resourceId]; - res.type('json').send({ - version: '1.0', - type: mimetype.includes('video') ? 'video' : 'photo', - author_url: opengraph.authorUrl, - provider_url: opengraph.providerUrl, - author_name: replaceholder(opengraph.author || '', data[resourceId]), - provider_name: replaceholder(opengraph.provider || '', data[resourceId]) - }); -}); +router.get('/oembed', (req, res, next) => + data.get(req.ass.resourceId) + .then(({ opengraph, mimetype, size, timestamp, originalname }) => + res.type('json').send({ + version: '1.0', + type: mimetype.includes('video') ? 'video' : 'photo', + author_url: opengraph.authorUrl, + provider_url: opengraph.providerUrl, + author_name: replaceholder(opengraph.author || '', size, timestamp, originalname), + provider_name: replaceholder(opengraph.provider || '', size, timestamp, originalname) + })) + .catch(next)); // Delete file -router.get('/delete/:deleteId', (req, res) => { - const { resourceId } = req.ass; - const deleteId = escape(req.params.deleteId); - const fileData = data[resourceId]; - - // If the delete ID doesn't match, don't delete the file - if (deleteId !== fileData.deleteId) return res.sendStatus(CODE_UNAUTHORIZED); - - // If the ID is invalid, return 400 because we are unable to process the resource - if (!resourceId || !fileData) return res.sendStatus(CODE_BAD_REQUEST); - - log(`Deleted: ${fileData.originalname} (${fileData.mimetype})`); - - // Save the file information - Promise.all([s3enabled ? deleteS3(fileData) : fs.rmSync(path(fileData.path)), fs.rmSync(path(diskFilePath, 'thumbnails/', fileData.thumbnail))]) - .then(() => { - delete data[resourceId]; - saveData(data); - res.type('text').send('File has been deleted!'); +router.get('/delete/:deleteId', (req, res, next) => { + let oldName, oldType; + data.get(req.ass.resourceId) + .then((fileData) => { + // Extract info for logs + oldName = fileData.originalname; + oldType = fileData.mimetype; + + // Clean deleteId + const deleteId = escape(req.params.deleteId); + + // If the delete ID doesn't match, don't delete the file + if (deleteId !== fileData.deleteId) return res.sendStatus(CODE_UNAUTHORIZED); + + // Save the file information + return Promise.all([s3enabled ? deleteS3(fileData) : fs.rmSync(path(fileData.path)), fs.rmSync(path(diskFilePath, 'thumbnails/', fileData.thumbnail))]); }) - .catch(console.error); + .then(() => data.del(req.ass.resourceId)) + .then(() => (log(`Deleted: ${oldName} (${oldType})`), res.type('text').send('File has been deleted!'))) + .catch(next); }); module.exports = router; diff --git a/routers/upload.js b/routers/upload.js index df50c44..1e84fc9 100644 --- a/routers/upload.js +++ b/routers/upload.js @@ -5,7 +5,7 @@ const { DateTime } = require('luxon'); const { WebhookClient, MessageEmbed } = require('discord.js'); const { doUpload, processUploaded } = require('../storage'); const { maxUploadSize, resourceIdSize, gfyIdSize, resourceIdType } = require('../config.json'); -const { path, saveData, log, verify, getTrueHttp, getTrueDomain, generateId, formatBytes } = require('../utils'); +const { path, log, verify, getTrueHttp, getTrueDomain, generateId, formatBytes } = require('../utils'); const { CODE_UNAUTHORIZED, CODE_PAYLOAD_TOO_LARGE } = require('../MagicNumbers.json'); const data = require('../data'); const users = require('../auth'); @@ -40,7 +40,7 @@ router.post('/', doUpload, processUploaded, ({ next }) => next()); router.use('/', (err, _req, res, next) => err.code && err.code === 'LIMIT_FILE_SIZE' ? res.status(CODE_PAYLOAD_TOO_LARGE).send(`Max upload size: ${maxUploadSize}MB`) : next(err)); // skipcq: JS-0229 // Process uploaded file -router.post('/', (req, res) => { +router.post('/', (req, res, next) => { // Load overrides const trueDomain = getTrueDomain(req.headers['x-ass-domain']); const generator = req.headers['x-ass-access'] || resourceIdType; @@ -67,54 +67,53 @@ router.post('/', (req, res) => { // Save the file information const resourceId = generateId(generator, resourceIdSize, req.headers['x-ass-gfycat'] || gfyIdSize, req.file.originalname); - data[resourceId.split('.')[0]] = req.file; - saveData(data); - - // Log the upload - const logInfo = `${req.file.originalname} (${req.file.mimetype})`; - log(`Uploaded: ${logInfo} (user: ${users[req.token] ? users[req.token].username : ''})`); - - // Build the URLs - const resourceUrl = `${getTrueHttp()}${trueDomain}/${resourceId}`; - const thumbnailUrl = `${getTrueHttp()}${trueDomain}/${resourceId}/thumbnail`; - const deleteUrl = `${getTrueHttp()}${trueDomain}/${resourceId}/delete/${req.file.deleteId}`; - - // Send the response - res.type('json').send({ resource: resourceUrl, thumbnail: thumbnailUrl, delete: deleteUrl }) - .on('finish', () => { - - // After we have sent the user the response, also send a Webhook to Discord (if headers are present) - if (req.headers['x-ass-webhook-client'] && req.headers['x-ass-webhook-token']) { - - // Build the webhook client & embed - const whc = new WebhookClient(req.headers['x-ass-webhook-client'], req.headers['x-ass-webhook-token']); - const embed = new MessageEmbed() - .setTitle(logInfo) - .setURL(resourceUrl) - .setDescription(`**Size:** \`${formatBytes(req.file.size)}\`\n**[Delete](${deleteUrl})**`) - .setThumbnail(thumbnailUrl) - .setColor(req.file.vibrant) - .setTimestamp(req.file.timestamp); - - // Send the embed to the webhook, then delete the client after to free resources - whc.send(null, { - username: req.headers['x-ass-webhook-username'] || 'ass', - avatarURL: req.headers['x-ass-webhook-avatar'] || ASS_LOGO, - embeds: [embed] - }).then(() => whc.destroy()); - } - - // Also update the users upload count - if (!users[req.token]) { - const generateUsername = () => generateId('random', 20, null); // skipcq: JS-0074 - let username = generateUsername(); - while (Object.values(users).findIndex((user) => user.username === username) !== -1) // skipcq: JS-0073 - username = generateUsername(); - users[req.token] = { username, count: 0 }; - } - users[req.token].count += 1; - fs.writeJsonSync(path('auth.json'), { users }, { spaces: 4 }) - }); + data.put(resourceId.split('.')[0], req.file).then(() => { + // Log the upload + const logInfo = `${req.file.originalname} (${req.file.mimetype})`; + log(`Uploaded: ${logInfo} (user: ${users[req.token] ? users[req.token].username : ''})`); + + // Build the URLs + const resourceUrl = `${getTrueHttp()}${trueDomain}/${resourceId}`; + const thumbnailUrl = `${getTrueHttp()}${trueDomain}/${resourceId}/thumbnail`; + const deleteUrl = `${getTrueHttp()}${trueDomain}/${resourceId}/delete/${req.file.deleteId}`; + + // Send the response + res.type('json').send({ resource: resourceUrl, thumbnail: thumbnailUrl, delete: deleteUrl }) + .on('finish', () => { + + // After we have sent the user the response, also send a Webhook to Discord (if headers are present) + if (req.headers['x-ass-webhook-client'] && req.headers['x-ass-webhook-token']) { + + // Build the webhook client & embed + const whc = new WebhookClient(req.headers['x-ass-webhook-client'], req.headers['x-ass-webhook-token']); + const embed = new MessageEmbed() + .setTitle(logInfo) + .setURL(resourceUrl) + .setDescription(`**Size:** \`${formatBytes(req.file.size)}\`\n**[Delete](${deleteUrl})**`) + .setThumbnail(thumbnailUrl) + .setColor(req.file.vibrant) + .setTimestamp(req.file.timestamp); + + // Send the embed to the webhook, then delete the client after to free resources + whc.send(null, { + username: req.headers['x-ass-webhook-username'] || 'ass', + avatarURL: req.headers['x-ass-webhook-avatar'] || ASS_LOGO, + embeds: [embed] + }).then(() => whc.destroy()); + } + + // Also update the users upload count + if (!users[req.token]) { + const generateUsername = () => generateId('random', 20, null); // skipcq: JS-0074 + let username = generateUsername(); + while (Object.values(users).findIndex((user) => user.username === username) !== -1) // skipcq: JS-0073 + username = generateUsername(); + users[req.token] = { username, count: 0 }; + } + users[req.token].count += 1; + fs.writeJsonSync(path('auth.json'), { users }, { spaces: 4 }) + }); + }).catch(next); }); module.exports = router; diff --git a/utils.js b/utils.js index 1c1e490..ae5c805 100755 --- a/utils.js +++ b/utils.js @@ -61,7 +61,7 @@ function formatBytes(bytes, decimals = 2) { // skipcq: JS-0074 return parseFloat((bytes / Math.pow(KILOBYTES, i)).toFixed(decimals < 0 ? 0 : decimals)).toString().concat(` ${sizes[i]}`); } -function replaceholder(data, { size, timestamp, originalname }) { +function replaceholder(data, size, timestamp, originalname) { return data .replace(/&size/g, formatBytes(size)) .replace(/&filename/g, originalname) @@ -105,7 +105,6 @@ module.exports = { randomHexColour, sanitize, log: console.log, - saveData: (data) => fs.writeJsonSync(Path.join(__dirname, 'data.json'), data, { spaces: 4 }), verify: (req, users) => req.headers.authorization && Object.prototype.hasOwnProperty.call(users, req.headers.authorization), renameFile: (req, newName) => new Promise((resolve, reject) => { try {