migrated to new StorageEngine system

pull/22/head
tycrek 3 years ago
parent 79d30cd576
commit 4966834609
No known key found for this signature in database
GPG Key ID: 25D74F3943625263

@ -3,7 +3,6 @@
"HTTPS": 443,
"CODE_OK": 200,
"CODE_NO_CONTENT": 204,
"CODE_BAD_REQUEST": 400,
"CODE_UNAUTHORIZED": 401,
"CODE_NOT_FOUND": 404,
"CODE_PAYLOAD_TOO_LARGE": 413,

@ -1 +1 @@
Subproject commit 43c8082e78d01d26f2e6c944e73bca67bb1d5197
Subproject commit 7e27c22ce5ac86e19789d7f94e85ad4225ea0b0a

@ -71,4 +71,4 @@ app.use(([err, , res,]) => {
});
// Host the server
app.listen(port, host, () => log(`Server started on [${host}:${port}]\nAuthorized users: ${Object.keys(users).length}\nAvailable files: ${Object.keys(data).length}`));
app.listen(port, host, () => log(`Server started on [${host}:${port}]\nAuthorized users: ${Object.keys(users).length}\nAvailable files: ${data.size}`));

@ -2,14 +2,6 @@
* Used for global data management
*/
const fs = require('fs-extra');
const { log, path } = require('./utils');
// Make sure data.json exists
if (!fs.existsSync(path('data.json'))) {
fs.writeJsonSync(path('data.json'), {}, { spaces: 4 });
log('File [data.json] created');
} else log('File [data.json] exists');
const data = require('./data.json');
const { JsonStorageEngine } = require('@tycrek/ass-storage-engine');
const data = new JsonStorageEngine();
module.exports = data;

50
package-lock.json generated

@ -9,6 +9,7 @@
"version": "0.6.0",
"license": "ISC",
"dependencies": {
"@tycrek/ass-storage-engine": "0.2.4",
"any-shell-escape": "^0.1.1",
"aws-sdk": "^2.930.0",
"check-node-version": "^4.1.0",
@ -560,6 +561,35 @@
"regenerator-runtime": "^0.13.3"
}
},
"node_modules/@tycrek/ass-storage-engine": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/@tycrek/ass-storage-engine/-/ass-storage-engine-0.2.4.tgz",
"integrity": "sha512-2LnfS1MQPKdB9Zb4e4oQcB+Pw8Z0+Q/mjoKowAH767WTxP3s1jD8yPFpTFOSRAoz52lrHJBMAEK31iIxqPBL/w==",
"dependencies": {
"fs-extra": "^10.0.0"
},
"engines": {
"node": "^14.x.x",
"npm": "^7.x.x"
},
"funding": {
"type": "patreon",
"url": "https://patreon.com/tycrek"
}
},
"node_modules/@tycrek/ass-storage-engine/node_modules/fs-extra": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz",
"integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==",
"dependencies": {
"graceful-fs": "^4.2.0",
"jsonfile": "^6.0.1",
"universalify": "^2.0.0"
},
"engines": {
"node": ">=12"
}
},
"node_modules/@types/node": {
"version": "10.17.60",
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz",
@ -3754,6 +3784,26 @@
"regenerator-runtime": "^0.13.3"
}
},
"@tycrek/ass-storage-engine": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/@tycrek/ass-storage-engine/-/ass-storage-engine-0.2.4.tgz",
"integrity": "sha512-2LnfS1MQPKdB9Zb4e4oQcB+Pw8Z0+Q/mjoKowAH767WTxP3s1jD8yPFpTFOSRAoz52lrHJBMAEK31iIxqPBL/w==",
"requires": {
"fs-extra": "^10.0.0"
},
"dependencies": {
"fs-extra": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz",
"integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==",
"requires": {
"graceful-fs": "^4.2.0",
"jsonfile": "^6.0.1",
"universalify": "^2.0.0"
}
}
}
},
"@types/node": {
"version": "10.17.60",
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz",

@ -1,6 +1,6 @@
{
"name": "ass",
"version": "0.6.0",
"version": "0.7.0",
"description": "The superior self-hosted ShareX server",
"main": "ass.js",
"engines": {
@ -34,6 +34,7 @@
"url": "https://patreon.com/tycrek"
},
"dependencies": {
"@tycrek/ass-storage-engine": "0.2.4",
"any-shell-escape": "^0.1.1",
"aws-sdk": "^2.930.0",
"check-node-version": "^4.1.0",

@ -3,8 +3,8 @@ const escape = require('escape-html');
const fetch = require('node-fetch');
const { deleteS3 } = require('../storage');
const { diskFilePath, s3enabled } = require('../config.json');
const { path, saveData, log, getTrueHttp, getTrueDomain, formatBytes, formatTimestamp, getS3url, getDirectUrl, getSafeExt, getResourceColor, replaceholder } = require('../utils');
const { CODE_BAD_REQUEST, CODE_UNAUTHORIZED, CODE_NOT_FOUND, } = require('../MagicNumbers.json');
const { path, log, getTrueHttp, getTrueDomain, formatBytes, formatTimestamp, getS3url, getDirectUrl, getSafeExt, getResourceColor, replaceholder } = require('../utils');
const { CODE_UNAUTHORIZED, CODE_NOT_FOUND, } = require('../MagicNumbers.json');
const data = require('../data');
const users = require('../auth');
@ -14,16 +14,17 @@ const router = express.Router();
// Middleware for parsing the resource ID and handling 404
router.use((req, res, next) => {
// Parse the resource ID
req.ass = { resourceId: escape(req.resourceId).split('.')[0] };
req.ass = { resourceId: escape(req.resourceId || '').split('.')[0] };
// If the ID is invalid, return 404. Otherwise, continue normally // skipcq: JS-0093
(!req.ass.resourceId || !data[req.ass.resourceId]) ? res.sendStatus(CODE_NOT_FOUND) : next();
// If the ID is invalid, return 404. Otherwise, continue normally
data.has(req.ass.resourceId)
.then((has) => has ? next() : res.sendStatus(CODE_NOT_FOUND))
.catch(next);
});
// View file
router.get('/', (req, res) => {
router.get('/', (req, res, next) => data.get(req.ass.resourceId).then((fileData) => {
const { resourceId } = req.ass;
const fileData = data[resourceId];
const isVideo = fileData.mimetype.includes('video');
// Build OpenGraph meta tags
@ -47,15 +48,12 @@ router.get('/', (req, res) => {
oembedUrl: `${getTrueHttp()}${getTrueDomain()}/${resourceId}/oembed`,
ogtype: isVideo ? 'video.other' : 'image',
urlType: `og:${isVideo ? 'video' : 'image'}`,
opengraph: replaceholder(ogs.join('\n'), fileData)
opengraph: replaceholder(ogs.join('\n'), fileData.size, fileData.timestamp, fileData.originalname)
});
});
}).catch(next));
// Direct resource
router.get('/direct*', (req, res) => {
const { resourceId } = req.ass;
const fileData = data[resourceId];
router.get('/direct*', (req, res, next) => data.get(req.ass.resourceId).then((fileData) => {
// Send file as an attachement for downloads
if (req.query.download)
res.header('Content-Disposition', `attachment; filename="${fileData.originalname}"`);
@ -73,58 +71,52 @@ router.get('/direct*', (req, res) => {
};
uploaders[s3enabled ? 's3' : 'local']();
});
}).catch(next));
// Thumbnail response
router.get('/thumbnail', (req, res) => {
const { resourceId } = req.ass;
// Read the file and send it to the client
fs.readFile(path(diskFilePath, 'thumbnails/', data[resourceId].thumbnail))
router.get('/thumbnail', (req, res, next) =>
data.get(req.ass.resourceId)
.then(({ thumbnail }) => fs.readFile(path(diskFilePath, 'thumbnails/', thumbnail)))
.then((fileData) => res.type('jpg').send(fileData))
.catch(console.error);
});
.catch(next));
// oEmbed response for clickable authors/providers
// https://oembed.com/
// https://old.reddit.com/r/discordapp/comments/82p8i6/a_basic_tutorial_on_how_to_get_the_most_out_of/
router.get('/oembed', (req, res) => {
const { resourceId } = req.ass;
// Build the oEmbed object & send the response
const { opengraph, mimetype } = data[resourceId];
res.type('json').send({
version: '1.0',
type: mimetype.includes('video') ? 'video' : 'photo',
author_url: opengraph.authorUrl,
provider_url: opengraph.providerUrl,
author_name: replaceholder(opengraph.author || '', data[resourceId]),
provider_name: replaceholder(opengraph.provider || '', data[resourceId])
});
});
router.get('/oembed', (req, res, next) =>
data.get(req.ass.resourceId)
.then(({ opengraph, mimetype, size, timestamp, originalname }) =>
res.type('json').send({
version: '1.0',
type: mimetype.includes('video') ? 'video' : 'photo',
author_url: opengraph.authorUrl,
provider_url: opengraph.providerUrl,
author_name: replaceholder(opengraph.author || '', size, timestamp, originalname),
provider_name: replaceholder(opengraph.provider || '', size, timestamp, originalname)
}))
.catch(next));
// Delete file
router.get('/delete/:deleteId', (req, res) => {
const { resourceId } = req.ass;
const deleteId = escape(req.params.deleteId);
const fileData = data[resourceId];
// If the delete ID doesn't match, don't delete the file
if (deleteId !== fileData.deleteId) return res.sendStatus(CODE_UNAUTHORIZED);
// If the ID is invalid, return 400 because we are unable to process the resource
if (!resourceId || !fileData) return res.sendStatus(CODE_BAD_REQUEST);
log(`Deleted: ${fileData.originalname} (${fileData.mimetype})`);
// Save the file information
Promise.all([s3enabled ? deleteS3(fileData) : fs.rmSync(path(fileData.path)), fs.rmSync(path(diskFilePath, 'thumbnails/', fileData.thumbnail))])
.then(() => {
delete data[resourceId];
saveData(data);
res.type('text').send('File has been deleted!');
router.get('/delete/:deleteId', (req, res, next) => {
let oldName, oldType;
data.get(req.ass.resourceId)
.then((fileData) => {
// Extract info for logs
oldName = fileData.originalname;
oldType = fileData.mimetype;
// Clean deleteId
const deleteId = escape(req.params.deleteId);
// If the delete ID doesn't match, don't delete the file
if (deleteId !== fileData.deleteId) return res.sendStatus(CODE_UNAUTHORIZED);
// Save the file information
return Promise.all([s3enabled ? deleteS3(fileData) : fs.rmSync(path(fileData.path)), fs.rmSync(path(diskFilePath, 'thumbnails/', fileData.thumbnail))]);
})
.catch(console.error);
.then(() => data.del(req.ass.resourceId))
.then(() => (log(`Deleted: ${oldName} (${oldType})`), res.type('text').send('File has been deleted!')))
.catch(next);
});
module.exports = router;

@ -5,7 +5,7 @@ const { DateTime } = require('luxon');
const { WebhookClient, MessageEmbed } = require('discord.js');
const { doUpload, processUploaded } = require('../storage');
const { maxUploadSize, resourceIdSize, gfyIdSize, resourceIdType } = require('../config.json');
const { path, saveData, log, verify, getTrueHttp, getTrueDomain, generateId, formatBytes } = require('../utils');
const { path, log, verify, getTrueHttp, getTrueDomain, generateId, formatBytes } = require('../utils');
const { CODE_UNAUTHORIZED, CODE_PAYLOAD_TOO_LARGE } = require('../MagicNumbers.json');
const data = require('../data');
const users = require('../auth');
@ -40,7 +40,7 @@ router.post('/', doUpload, processUploaded, ({ next }) => next());
router.use('/', (err, _req, res, next) => err.code && err.code === 'LIMIT_FILE_SIZE' ? res.status(CODE_PAYLOAD_TOO_LARGE).send(`Max upload size: ${maxUploadSize}MB`) : next(err)); // skipcq: JS-0229
// Process uploaded file
router.post('/', (req, res) => {
router.post('/', (req, res, next) => {
// Load overrides
const trueDomain = getTrueDomain(req.headers['x-ass-domain']);
const generator = req.headers['x-ass-access'] || resourceIdType;
@ -67,54 +67,53 @@ router.post('/', (req, res) => {
// Save the file information
const resourceId = generateId(generator, resourceIdSize, req.headers['x-ass-gfycat'] || gfyIdSize, req.file.originalname);
data[resourceId.split('.')[0]] = req.file;
saveData(data);
// Log the upload
const logInfo = `${req.file.originalname} (${req.file.mimetype})`;
log(`Uploaded: ${logInfo} (user: ${users[req.token] ? users[req.token].username : '<token-only>'})`);
// Build the URLs
const resourceUrl = `${getTrueHttp()}${trueDomain}/${resourceId}`;
const thumbnailUrl = `${getTrueHttp()}${trueDomain}/${resourceId}/thumbnail`;
const deleteUrl = `${getTrueHttp()}${trueDomain}/${resourceId}/delete/${req.file.deleteId}`;
// Send the response
res.type('json').send({ resource: resourceUrl, thumbnail: thumbnailUrl, delete: deleteUrl })
.on('finish', () => {
// After we have sent the user the response, also send a Webhook to Discord (if headers are present)
if (req.headers['x-ass-webhook-client'] && req.headers['x-ass-webhook-token']) {
// Build the webhook client & embed
const whc = new WebhookClient(req.headers['x-ass-webhook-client'], req.headers['x-ass-webhook-token']);
const embed = new MessageEmbed()
.setTitle(logInfo)
.setURL(resourceUrl)
.setDescription(`**Size:** \`${formatBytes(req.file.size)}\`\n**[Delete](${deleteUrl})**`)
.setThumbnail(thumbnailUrl)
.setColor(req.file.vibrant)
.setTimestamp(req.file.timestamp);
// Send the embed to the webhook, then delete the client after to free resources
whc.send(null, {
username: req.headers['x-ass-webhook-username'] || 'ass',
avatarURL: req.headers['x-ass-webhook-avatar'] || ASS_LOGO,
embeds: [embed]
}).then(() => whc.destroy());
}
// Also update the users upload count
if (!users[req.token]) {
const generateUsername = () => generateId('random', 20, null); // skipcq: JS-0074
let username = generateUsername();
while (Object.values(users).findIndex((user) => user.username === username) !== -1) // skipcq: JS-0073
username = generateUsername();
users[req.token] = { username, count: 0 };
}
users[req.token].count += 1;
fs.writeJsonSync(path('auth.json'), { users }, { spaces: 4 })
});
data.put(resourceId.split('.')[0], req.file).then(() => {
// Log the upload
const logInfo = `${req.file.originalname} (${req.file.mimetype})`;
log(`Uploaded: ${logInfo} (user: ${users[req.token] ? users[req.token].username : '<token-only>'})`);
// Build the URLs
const resourceUrl = `${getTrueHttp()}${trueDomain}/${resourceId}`;
const thumbnailUrl = `${getTrueHttp()}${trueDomain}/${resourceId}/thumbnail`;
const deleteUrl = `${getTrueHttp()}${trueDomain}/${resourceId}/delete/${req.file.deleteId}`;
// Send the response
res.type('json').send({ resource: resourceUrl, thumbnail: thumbnailUrl, delete: deleteUrl })
.on('finish', () => {
// After we have sent the user the response, also send a Webhook to Discord (if headers are present)
if (req.headers['x-ass-webhook-client'] && req.headers['x-ass-webhook-token']) {
// Build the webhook client & embed
const whc = new WebhookClient(req.headers['x-ass-webhook-client'], req.headers['x-ass-webhook-token']);
const embed = new MessageEmbed()
.setTitle(logInfo)
.setURL(resourceUrl)
.setDescription(`**Size:** \`${formatBytes(req.file.size)}\`\n**[Delete](${deleteUrl})**`)
.setThumbnail(thumbnailUrl)
.setColor(req.file.vibrant)
.setTimestamp(req.file.timestamp);
// Send the embed to the webhook, then delete the client after to free resources
whc.send(null, {
username: req.headers['x-ass-webhook-username'] || 'ass',
avatarURL: req.headers['x-ass-webhook-avatar'] || ASS_LOGO,
embeds: [embed]
}).then(() => whc.destroy());
}
// Also update the users upload count
if (!users[req.token]) {
const generateUsername = () => generateId('random', 20, null); // skipcq: JS-0074
let username = generateUsername();
while (Object.values(users).findIndex((user) => user.username === username) !== -1) // skipcq: JS-0073
username = generateUsername();
users[req.token] = { username, count: 0 };
}
users[req.token].count += 1;
fs.writeJsonSync(path('auth.json'), { users }, { spaces: 4 })
});
}).catch(next);
});
module.exports = router;

@ -61,7 +61,7 @@ function formatBytes(bytes, decimals = 2) { // skipcq: JS-0074
return parseFloat((bytes / Math.pow(KILOBYTES, i)).toFixed(decimals < 0 ? 0 : decimals)).toString().concat(` ${sizes[i]}`);
}
function replaceholder(data, { size, timestamp, originalname }) {
function replaceholder(data, size, timestamp, originalname) {
return data
.replace(/&size/g, formatBytes(size))
.replace(/&filename/g, originalname)
@ -105,7 +105,6 @@ module.exports = {
randomHexColour,
sanitize,
log: console.log,
saveData: (data) => fs.writeJsonSync(Path.join(__dirname, 'data.json'), data, { spaces: 4 }),
verify: (req, users) => req.headers.authorization && Object.prototype.hasOwnProperty.call(users, req.headers.authorization),
renameFile: (req, newName) => new Promise((resolve, reject) => {
try {

Loading…
Cancel
Save