From 15cf7c0e51c20ab1aa78daca54399fed151710a2 Mon Sep 17 00:00:00 2001 From: tycrek Date: Mon, 5 Jul 2021 22:05:08 -0600 Subject: [PATCH 1/7] added notes to readme about upcoming storage engines --- README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0904dd9..153b6b4 100755 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ If you primarily share media on Discord, you can add these additional (optional) | **`X-Ass-OG-Author-Url`** | URL to open when the Author is clicked | | **`X-Ass-OG-Provider`** | Smaller text shown above the author | | **`X-Ass-OG-Provider-Url`** | URL to open when the Provider is clicked | -| **`X-Ass-OG-Color`** | Colour shown on the left side of the embed. Must be one of `&random`, `&vibrant`, or a hex colour value (for example: `#fe3c29`). Random is a randomly generated hex value and Vibrant is sourced from the image itself | +| **`X-Ass-OG-Color`** | Colour shown on the left side of the embed. Must be one of `&random`, `&vibrant`, or a hex colour value (for example: `#fe3c29`). Random is a randomly generated hex value & Vibrant is sourced from the image itself | #### Embed placeholders @@ -178,6 +178,23 @@ Now you should see `My awesome dashboard!` when you navigate to `http://your-ass **For a detailed walkthrough on developing your first frontend, [consult the wiki](https://github.com/tycrek/ass/wiki/Writing-a-custom-frontend).** +## Storage engines (WIP) + +***This feature is currently a work-in-progress & may not work.*** + +Storage engines are responsible for managing your data. "Data" is anything that has two or more parts: an identifier & the actual data itself. With ass, the data is a JSON object representing the uploaded resource. The identifier is the unique ID in the URL returned to the user on upload. + +ass aims to support these storage methods at a minimum: + +- **JSON** +- **Mongo** (soon) +- **MySQL** (soon) +- **PostgreSQL** (soon) + +An ass storage engine implements support for one type of database (or file, such as JSON or YAML). This lets developers & system administrators pick their database of choice, because all they'll have to do is plugin the connection/authentication details, then ass will handle the rest, using the resource ID as the key. + +The only storage engine ass comes with by default is **JSON**. Others will be published to [npm](https://www.npmjs.com/). + ## Flameshot users (Linux) Use [this script](https://github.com/tycrek/ass/blob/master/flameshot_example.sh) kindly provided by [@ToxicAven](https://github.com/ToxicAven). For the `KEY`, put your token. From 79d30cd576a01924cb6f74c49241ab0aa9b288fc Mon Sep 17 00:00:00 2001 From: tycrek Date: Mon, 5 Jul 2021 22:14:53 -0600 Subject: [PATCH 2/7] added link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 153b6b4..b56a6e3 100755 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ Now you should see `My awesome dashboard!` when you navigate to `http://your-ass ***This feature is currently a work-in-progress & may not work.*** -Storage engines are responsible for managing your data. "Data" is anything that has two or more parts: an identifier & the actual data itself. With ass, the data is a JSON object representing the uploaded resource. The identifier is the unique ID in the URL returned to the user on upload. +[Storage engines](https://github.com/tycrek/ass-storage-engine) are responsible for managing your data. "Data" is anything that has two or more parts: an identifier & the actual data itself. With ass, the data is a JSON object representing the uploaded resource. The identifier is the unique ID in the URL returned to the user on upload. ass aims to support these storage methods at a minimum: From 49668346096bdacd8e764ce028eb30b3bf940b66 Mon Sep 17 00:00:00 2001 From: tycrek Date: Tue, 6 Jul 2021 13:55:49 -0600 Subject: [PATCH 3/7] migrated to new StorageEngine system --- MagicNumbers.json | 1 - ass-x | 2 +- ass.js | 2 +- data.js | 12 +----- package-lock.json | 50 ++++++++++++++++++++++ package.json | 3 +- routers/resource.js | 102 ++++++++++++++++++++------------------------ routers/upload.js | 99 +++++++++++++++++++++--------------------- utils.js | 3 +- 9 files changed, 153 insertions(+), 121 deletions(-) diff --git a/MagicNumbers.json b/MagicNumbers.json index 074de1b..842c5e0 100644 --- a/MagicNumbers.json +++ b/MagicNumbers.json @@ -3,7 +3,6 @@ "HTTPS": 443, "CODE_OK": 200, "CODE_NO_CONTENT": 204, - "CODE_BAD_REQUEST": 400, "CODE_UNAUTHORIZED": 401, "CODE_NOT_FOUND": 404, "CODE_PAYLOAD_TOO_LARGE": 413, diff --git a/ass-x b/ass-x index 43c8082..7e27c22 160000 --- a/ass-x +++ b/ass-x @@ -1 +1 @@ -Subproject commit 43c8082e78d01d26f2e6c944e73bca67bb1d5197 +Subproject commit 7e27c22ce5ac86e19789d7f94e85ad4225ea0b0a diff --git a/ass.js b/ass.js index 70d95c5..72518cc 100755 --- a/ass.js +++ b/ass.js @@ -71,4 +71,4 @@ app.use(([err, , res,]) => { }); // Host the server -app.listen(port, host, () => log(`Server started on [${host}:${port}]\nAuthorized users: ${Object.keys(users).length}\nAvailable files: ${Object.keys(data).length}`)); +app.listen(port, host, () => log(`Server started on [${host}:${port}]\nAuthorized users: ${Object.keys(users).length}\nAvailable files: ${data.size}`)); diff --git a/data.js b/data.js index c128362..e28d932 100644 --- a/data.js +++ b/data.js @@ -2,14 +2,6 @@ * Used for global data management */ -const fs = require('fs-extra'); -const { log, path } = require('./utils'); - -// Make sure data.json exists -if (!fs.existsSync(path('data.json'))) { - fs.writeJsonSync(path('data.json'), {}, { spaces: 4 }); - log('File [data.json] created'); -} else log('File [data.json] exists'); - -const data = require('./data.json'); +const { JsonStorageEngine } = require('@tycrek/ass-storage-engine'); +const data = new JsonStorageEngine(); module.exports = data; diff --git a/package-lock.json b/package-lock.json index 2b2ec62..4acd3a4 100755 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.6.0", "license": "ISC", "dependencies": { + "@tycrek/ass-storage-engine": "0.2.4", "any-shell-escape": "^0.1.1", "aws-sdk": "^2.930.0", "check-node-version": "^4.1.0", @@ -560,6 +561,35 @@ "regenerator-runtime": "^0.13.3" } }, + "node_modules/@tycrek/ass-storage-engine": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@tycrek/ass-storage-engine/-/ass-storage-engine-0.2.4.tgz", + "integrity": "sha512-2LnfS1MQPKdB9Zb4e4oQcB+Pw8Z0+Q/mjoKowAH767WTxP3s1jD8yPFpTFOSRAoz52lrHJBMAEK31iIxqPBL/w==", + "dependencies": { + "fs-extra": "^10.0.0" + }, + "engines": { + "node": "^14.x.x", + "npm": "^7.x.x" + }, + "funding": { + "type": "patreon", + "url": "https://patreon.com/tycrek" + } + }, + "node_modules/@tycrek/ass-storage-engine/node_modules/fs-extra": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz", + "integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/@types/node": { "version": "10.17.60", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", @@ -3754,6 +3784,26 @@ "regenerator-runtime": "^0.13.3" } }, + "@tycrek/ass-storage-engine": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@tycrek/ass-storage-engine/-/ass-storage-engine-0.2.4.tgz", + "integrity": "sha512-2LnfS1MQPKdB9Zb4e4oQcB+Pw8Z0+Q/mjoKowAH767WTxP3s1jD8yPFpTFOSRAoz52lrHJBMAEK31iIxqPBL/w==", + "requires": { + "fs-extra": "^10.0.0" + }, + "dependencies": { + "fs-extra": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz", + "integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==", + "requires": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + } + } + }, "@types/node": { "version": "10.17.60", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", diff --git a/package.json b/package.json index dd0c6df..e5393c0 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ass", - "version": "0.6.0", + "version": "0.7.0", "description": "The superior self-hosted ShareX server", "main": "ass.js", "engines": { @@ -34,6 +34,7 @@ "url": "https://patreon.com/tycrek" }, "dependencies": { + "@tycrek/ass-storage-engine": "0.2.4", "any-shell-escape": "^0.1.1", "aws-sdk": "^2.930.0", "check-node-version": "^4.1.0", diff --git a/routers/resource.js b/routers/resource.js index c4a24ec..f1b53b4 100644 --- a/routers/resource.js +++ b/routers/resource.js @@ -3,8 +3,8 @@ const escape = require('escape-html'); const fetch = require('node-fetch'); const { deleteS3 } = require('../storage'); const { diskFilePath, s3enabled } = require('../config.json'); -const { path, saveData, log, getTrueHttp, getTrueDomain, formatBytes, formatTimestamp, getS3url, getDirectUrl, getSafeExt, getResourceColor, replaceholder } = require('../utils'); -const { CODE_BAD_REQUEST, CODE_UNAUTHORIZED, CODE_NOT_FOUND, } = require('../MagicNumbers.json'); +const { path, log, getTrueHttp, getTrueDomain, formatBytes, formatTimestamp, getS3url, getDirectUrl, getSafeExt, getResourceColor, replaceholder } = require('../utils'); +const { CODE_UNAUTHORIZED, CODE_NOT_FOUND, } = require('../MagicNumbers.json'); const data = require('../data'); const users = require('../auth'); @@ -14,16 +14,17 @@ const router = express.Router(); // Middleware for parsing the resource ID and handling 404 router.use((req, res, next) => { // Parse the resource ID - req.ass = { resourceId: escape(req.resourceId).split('.')[0] }; + req.ass = { resourceId: escape(req.resourceId || '').split('.')[0] }; - // If the ID is invalid, return 404. Otherwise, continue normally // skipcq: JS-0093 - (!req.ass.resourceId || !data[req.ass.resourceId]) ? res.sendStatus(CODE_NOT_FOUND) : next(); + // If the ID is invalid, return 404. Otherwise, continue normally + data.has(req.ass.resourceId) + .then((has) => has ? next() : res.sendStatus(CODE_NOT_FOUND)) + .catch(next); }); // View file -router.get('/', (req, res) => { +router.get('/', (req, res, next) => data.get(req.ass.resourceId).then((fileData) => { const { resourceId } = req.ass; - const fileData = data[resourceId]; const isVideo = fileData.mimetype.includes('video'); // Build OpenGraph meta tags @@ -47,15 +48,12 @@ router.get('/', (req, res) => { oembedUrl: `${getTrueHttp()}${getTrueDomain()}/${resourceId}/oembed`, ogtype: isVideo ? 'video.other' : 'image', urlType: `og:${isVideo ? 'video' : 'image'}`, - opengraph: replaceholder(ogs.join('\n'), fileData) + opengraph: replaceholder(ogs.join('\n'), fileData.size, fileData.timestamp, fileData.originalname) }); -}); +}).catch(next)); // Direct resource -router.get('/direct*', (req, res) => { - const { resourceId } = req.ass; - const fileData = data[resourceId]; - +router.get('/direct*', (req, res, next) => data.get(req.ass.resourceId).then((fileData) => { // Send file as an attachement for downloads if (req.query.download) res.header('Content-Disposition', `attachment; filename="${fileData.originalname}"`); @@ -73,58 +71,52 @@ router.get('/direct*', (req, res) => { }; uploaders[s3enabled ? 's3' : 'local'](); -}); +}).catch(next)); // Thumbnail response -router.get('/thumbnail', (req, res) => { - const { resourceId } = req.ass; - - // Read the file and send it to the client - fs.readFile(path(diskFilePath, 'thumbnails/', data[resourceId].thumbnail)) +router.get('/thumbnail', (req, res, next) => + data.get(req.ass.resourceId) + .then(({ thumbnail }) => fs.readFile(path(diskFilePath, 'thumbnails/', thumbnail))) .then((fileData) => res.type('jpg').send(fileData)) - .catch(console.error); -}); + .catch(next)); // oEmbed response for clickable authors/providers // https://oembed.com/ // https://old.reddit.com/r/discordapp/comments/82p8i6/a_basic_tutorial_on_how_to_get_the_most_out_of/ -router.get('/oembed', (req, res) => { - const { resourceId } = req.ass; - - // Build the oEmbed object & send the response - const { opengraph, mimetype } = data[resourceId]; - res.type('json').send({ - version: '1.0', - type: mimetype.includes('video') ? 'video' : 'photo', - author_url: opengraph.authorUrl, - provider_url: opengraph.providerUrl, - author_name: replaceholder(opengraph.author || '', data[resourceId]), - provider_name: replaceholder(opengraph.provider || '', data[resourceId]) - }); -}); +router.get('/oembed', (req, res, next) => + data.get(req.ass.resourceId) + .then(({ opengraph, mimetype, size, timestamp, originalname }) => + res.type('json').send({ + version: '1.0', + type: mimetype.includes('video') ? 'video' : 'photo', + author_url: opengraph.authorUrl, + provider_url: opengraph.providerUrl, + author_name: replaceholder(opengraph.author || '', size, timestamp, originalname), + provider_name: replaceholder(opengraph.provider || '', size, timestamp, originalname) + })) + .catch(next)); // Delete file -router.get('/delete/:deleteId', (req, res) => { - const { resourceId } = req.ass; - const deleteId = escape(req.params.deleteId); - const fileData = data[resourceId]; - - // If the delete ID doesn't match, don't delete the file - if (deleteId !== fileData.deleteId) return res.sendStatus(CODE_UNAUTHORIZED); - - // If the ID is invalid, return 400 because we are unable to process the resource - if (!resourceId || !fileData) return res.sendStatus(CODE_BAD_REQUEST); - - log(`Deleted: ${fileData.originalname} (${fileData.mimetype})`); - - // Save the file information - Promise.all([s3enabled ? deleteS3(fileData) : fs.rmSync(path(fileData.path)), fs.rmSync(path(diskFilePath, 'thumbnails/', fileData.thumbnail))]) - .then(() => { - delete data[resourceId]; - saveData(data); - res.type('text').send('File has been deleted!'); +router.get('/delete/:deleteId', (req, res, next) => { + let oldName, oldType; + data.get(req.ass.resourceId) + .then((fileData) => { + // Extract info for logs + oldName = fileData.originalname; + oldType = fileData.mimetype; + + // Clean deleteId + const deleteId = escape(req.params.deleteId); + + // If the delete ID doesn't match, don't delete the file + if (deleteId !== fileData.deleteId) return res.sendStatus(CODE_UNAUTHORIZED); + + // Save the file information + return Promise.all([s3enabled ? deleteS3(fileData) : fs.rmSync(path(fileData.path)), fs.rmSync(path(diskFilePath, 'thumbnails/', fileData.thumbnail))]); }) - .catch(console.error); + .then(() => data.del(req.ass.resourceId)) + .then(() => (log(`Deleted: ${oldName} (${oldType})`), res.type('text').send('File has been deleted!'))) + .catch(next); }); module.exports = router; diff --git a/routers/upload.js b/routers/upload.js index df50c44..1e84fc9 100644 --- a/routers/upload.js +++ b/routers/upload.js @@ -5,7 +5,7 @@ const { DateTime } = require('luxon'); const { WebhookClient, MessageEmbed } = require('discord.js'); const { doUpload, processUploaded } = require('../storage'); const { maxUploadSize, resourceIdSize, gfyIdSize, resourceIdType } = require('../config.json'); -const { path, saveData, log, verify, getTrueHttp, getTrueDomain, generateId, formatBytes } = require('../utils'); +const { path, log, verify, getTrueHttp, getTrueDomain, generateId, formatBytes } = require('../utils'); const { CODE_UNAUTHORIZED, CODE_PAYLOAD_TOO_LARGE } = require('../MagicNumbers.json'); const data = require('../data'); const users = require('../auth'); @@ -40,7 +40,7 @@ router.post('/', doUpload, processUploaded, ({ next }) => next()); router.use('/', (err, _req, res, next) => err.code && err.code === 'LIMIT_FILE_SIZE' ? res.status(CODE_PAYLOAD_TOO_LARGE).send(`Max upload size: ${maxUploadSize}MB`) : next(err)); // skipcq: JS-0229 // Process uploaded file -router.post('/', (req, res) => { +router.post('/', (req, res, next) => { // Load overrides const trueDomain = getTrueDomain(req.headers['x-ass-domain']); const generator = req.headers['x-ass-access'] || resourceIdType; @@ -67,54 +67,53 @@ router.post('/', (req, res) => { // Save the file information const resourceId = generateId(generator, resourceIdSize, req.headers['x-ass-gfycat'] || gfyIdSize, req.file.originalname); - data[resourceId.split('.')[0]] = req.file; - saveData(data); - - // Log the upload - const logInfo = `${req.file.originalname} (${req.file.mimetype})`; - log(`Uploaded: ${logInfo} (user: ${users[req.token] ? users[req.token].username : ''})`); - - // Build the URLs - const resourceUrl = `${getTrueHttp()}${trueDomain}/${resourceId}`; - const thumbnailUrl = `${getTrueHttp()}${trueDomain}/${resourceId}/thumbnail`; - const deleteUrl = `${getTrueHttp()}${trueDomain}/${resourceId}/delete/${req.file.deleteId}`; - - // Send the response - res.type('json').send({ resource: resourceUrl, thumbnail: thumbnailUrl, delete: deleteUrl }) - .on('finish', () => { - - // After we have sent the user the response, also send a Webhook to Discord (if headers are present) - if (req.headers['x-ass-webhook-client'] && req.headers['x-ass-webhook-token']) { - - // Build the webhook client & embed - const whc = new WebhookClient(req.headers['x-ass-webhook-client'], req.headers['x-ass-webhook-token']); - const embed = new MessageEmbed() - .setTitle(logInfo) - .setURL(resourceUrl) - .setDescription(`**Size:** \`${formatBytes(req.file.size)}\`\n**[Delete](${deleteUrl})**`) - .setThumbnail(thumbnailUrl) - .setColor(req.file.vibrant) - .setTimestamp(req.file.timestamp); - - // Send the embed to the webhook, then delete the client after to free resources - whc.send(null, { - username: req.headers['x-ass-webhook-username'] || 'ass', - avatarURL: req.headers['x-ass-webhook-avatar'] || ASS_LOGO, - embeds: [embed] - }).then(() => whc.destroy()); - } - - // Also update the users upload count - if (!users[req.token]) { - const generateUsername = () => generateId('random', 20, null); // skipcq: JS-0074 - let username = generateUsername(); - while (Object.values(users).findIndex((user) => user.username === username) !== -1) // skipcq: JS-0073 - username = generateUsername(); - users[req.token] = { username, count: 0 }; - } - users[req.token].count += 1; - fs.writeJsonSync(path('auth.json'), { users }, { spaces: 4 }) - }); + data.put(resourceId.split('.')[0], req.file).then(() => { + // Log the upload + const logInfo = `${req.file.originalname} (${req.file.mimetype})`; + log(`Uploaded: ${logInfo} (user: ${users[req.token] ? users[req.token].username : ''})`); + + // Build the URLs + const resourceUrl = `${getTrueHttp()}${trueDomain}/${resourceId}`; + const thumbnailUrl = `${getTrueHttp()}${trueDomain}/${resourceId}/thumbnail`; + const deleteUrl = `${getTrueHttp()}${trueDomain}/${resourceId}/delete/${req.file.deleteId}`; + + // Send the response + res.type('json').send({ resource: resourceUrl, thumbnail: thumbnailUrl, delete: deleteUrl }) + .on('finish', () => { + + // After we have sent the user the response, also send a Webhook to Discord (if headers are present) + if (req.headers['x-ass-webhook-client'] && req.headers['x-ass-webhook-token']) { + + // Build the webhook client & embed + const whc = new WebhookClient(req.headers['x-ass-webhook-client'], req.headers['x-ass-webhook-token']); + const embed = new MessageEmbed() + .setTitle(logInfo) + .setURL(resourceUrl) + .setDescription(`**Size:** \`${formatBytes(req.file.size)}\`\n**[Delete](${deleteUrl})**`) + .setThumbnail(thumbnailUrl) + .setColor(req.file.vibrant) + .setTimestamp(req.file.timestamp); + + // Send the embed to the webhook, then delete the client after to free resources + whc.send(null, { + username: req.headers['x-ass-webhook-username'] || 'ass', + avatarURL: req.headers['x-ass-webhook-avatar'] || ASS_LOGO, + embeds: [embed] + }).then(() => whc.destroy()); + } + + // Also update the users upload count + if (!users[req.token]) { + const generateUsername = () => generateId('random', 20, null); // skipcq: JS-0074 + let username = generateUsername(); + while (Object.values(users).findIndex((user) => user.username === username) !== -1) // skipcq: JS-0073 + username = generateUsername(); + users[req.token] = { username, count: 0 }; + } + users[req.token].count += 1; + fs.writeJsonSync(path('auth.json'), { users }, { spaces: 4 }) + }); + }).catch(next); }); module.exports = router; diff --git a/utils.js b/utils.js index 1c1e490..ae5c805 100755 --- a/utils.js +++ b/utils.js @@ -61,7 +61,7 @@ function formatBytes(bytes, decimals = 2) { // skipcq: JS-0074 return parseFloat((bytes / Math.pow(KILOBYTES, i)).toFixed(decimals < 0 ? 0 : decimals)).toString().concat(` ${sizes[i]}`); } -function replaceholder(data, { size, timestamp, originalname }) { +function replaceholder(data, size, timestamp, originalname) { return data .replace(/&size/g, formatBytes(size)) .replace(/&filename/g, originalname) @@ -105,7 +105,6 @@ module.exports = { randomHexColour, sanitize, log: console.log, - saveData: (data) => fs.writeJsonSync(Path.join(__dirname, 'data.json'), data, { spaces: 4 }), verify: (req, users) => req.headers.authorization && Object.prototype.hasOwnProperty.call(users, req.headers.authorization), renameFile: (req, newName) => new Promise((resolve, reject) => { try { From 2493d61750343dd45bf8f56c495434dff48955d5 Mon Sep 17 00:00:00 2001 From: tycrek Date: Tue, 6 Jul 2021 14:01:14 -0600 Subject: [PATCH 4/7] fixed README for StorageEngines woops --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b56a6e3..33ca556 100755 --- a/README.md +++ b/README.md @@ -178,11 +178,9 @@ Now you should see `My awesome dashboard!` when you navigate to `http://your-ass **For a detailed walkthrough on developing your first frontend, [consult the wiki](https://github.com/tycrek/ass/wiki/Writing-a-custom-frontend).** -## Storage engines (WIP) +## StorageEngines -***This feature is currently a work-in-progress & may not work.*** - -[Storage engines](https://github.com/tycrek/ass-storage-engine) are responsible for managing your data. "Data" is anything that has two or more parts: an identifier & the actual data itself. With ass, the data is a JSON object representing the uploaded resource. The identifier is the unique ID in the URL returned to the user on upload. +[StorageEngines](https://github.com/tycrek/ass-storage-engine) are responsible for managing your data. "Data" has two parts: an identifier & the actual data itself. With ass, the data is a JSON object representing the uploaded resource. The identifier is the unique ID in the URL returned to the user on upload. ass aims to support these storage methods at a minimum: @@ -191,9 +189,11 @@ ass aims to support these storage methods at a minimum: - **MySQL** (soon) - **PostgreSQL** (soon) -An ass storage engine implements support for one type of database (or file, such as JSON or YAML). This lets developers & system administrators pick their database of choice, because all they'll have to do is plugin the connection/authentication details, then ass will handle the rest, using the resource ID as the key. +An ass StorageEngine implements support for one type of database (or file, such as JSON or YAML). This lets ass server hosts pick their database of choice, because all they'll have to do is plugin the connection/authentication details, then ass will handle the rest, using the resource ID as the key. + +The only storage engine ass comes with by default is **JSON**. Others will be published to [npm](https://www.npmjs.com/) and listed here. -The only storage engine ass comes with by default is **JSON**. Others will be published to [npm](https://www.npmjs.com/). +**A wiki page on writing a custom StorageEngine is coming soon. Once complete, you can find it [here](https://github.com/tycrek/ass/wiki/Writing-a-StorageEngine).** ## Flameshot users (Linux) From 39c1444b401b4eb154d9621b8558edba2973c93a Mon Sep 17 00:00:00 2001 From: tycrek Date: Tue, 6 Jul 2021 14:44:04 -0600 Subject: [PATCH 5/7] print StorageEngine name and type --- ass.js | 2 +- package-lock.json | 18 +++++++++--------- package.json | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ass.js b/ass.js index 72518cc..9771073 100755 --- a/ass.js +++ b/ass.js @@ -34,7 +34,7 @@ const ROUTERS = { // Read users and data const users = require('./auth'); const data = require('./data'); -log('Users & data read from filesystem'); +log(`StorageEngine: ${data.name} (${data.type})`); //#endregion // Create thumbnails directory diff --git a/package-lock.json b/package-lock.json index 4acd3a4..bc0b5b9 100755 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,15 @@ { "name": "ass", - "version": "0.6.0", + "version": "0.7.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "ass", - "version": "0.6.0", + "version": "0.7.0", "license": "ISC", "dependencies": { - "@tycrek/ass-storage-engine": "0.2.4", + "@tycrek/ass-storage-engine": "0.2.5", "any-shell-escape": "^0.1.1", "aws-sdk": "^2.930.0", "check-node-version": "^4.1.0", @@ -562,9 +562,9 @@ } }, "node_modules/@tycrek/ass-storage-engine": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/@tycrek/ass-storage-engine/-/ass-storage-engine-0.2.4.tgz", - "integrity": "sha512-2LnfS1MQPKdB9Zb4e4oQcB+Pw8Z0+Q/mjoKowAH767WTxP3s1jD8yPFpTFOSRAoz52lrHJBMAEK31iIxqPBL/w==", + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/@tycrek/ass-storage-engine/-/ass-storage-engine-0.2.5.tgz", + "integrity": "sha512-D4C4WAtTQIkoN1QH7l9h4gJTIWcrfpqZPd2nrgLc8O400eZdZJWkeA9Tf6UD18V5e5GWun3bwJFmCqfTwmmjWw==", "dependencies": { "fs-extra": "^10.0.0" }, @@ -3785,9 +3785,9 @@ } }, "@tycrek/ass-storage-engine": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/@tycrek/ass-storage-engine/-/ass-storage-engine-0.2.4.tgz", - "integrity": "sha512-2LnfS1MQPKdB9Zb4e4oQcB+Pw8Z0+Q/mjoKowAH767WTxP3s1jD8yPFpTFOSRAoz52lrHJBMAEK31iIxqPBL/w==", + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/@tycrek/ass-storage-engine/-/ass-storage-engine-0.2.5.tgz", + "integrity": "sha512-D4C4WAtTQIkoN1QH7l9h4gJTIWcrfpqZPd2nrgLc8O400eZdZJWkeA9Tf6UD18V5e5GWun3bwJFmCqfTwmmjWw==", "requires": { "fs-extra": "^10.0.0" }, diff --git a/package.json b/package.json index e5393c0..c25f44b 100755 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "url": "https://patreon.com/tycrek" }, "dependencies": { - "@tycrek/ass-storage-engine": "0.2.4", + "@tycrek/ass-storage-engine": "0.2.5", "any-shell-escape": "^0.1.1", "aws-sdk": "^2.930.0", "check-node-version": "^4.1.0", From 3446bc7da9a9fe398132466954184035be441f53 Mon Sep 17 00:00:00 2001 From: tycrek Date: Tue, 6 Jul 2021 14:44:56 -0600 Subject: [PATCH 6/7] Updated README with StorageEngine shazaam --- README.md | 49 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 33ca556..294cb6d 100755 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ - ✔️ Thumbnail support - ✔️ Basic multi-user support - ✔️ Configurable global upload limit (per-user coming soon!) +- ✔️ Basic macOS/Linux support using other clients including [Flameshot](https://flameshot.org/) ([ass-compatible Flameshot script](https://github.com/tycrek/ass#flameshot-users-linux)) & [MagicCap](https://magiccap.me/) - ✔️ Local storage *or* block-storage support for [Amazon S3](https://aws.amazon.com/s3/) (including [DigitalOcean Spaces](https://www.digitalocean.com/products/spaces/)) - ✔️ Custom pluggable frontend dashboards using [Git Submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) - ✔️ Multiple access types @@ -36,11 +37,14 @@ - **Mixed-case alphanumeric** - **Gfycat** - **Original** -- ❌ Multiple database types - - **JSON** - - **Mongo** (soon!) - - **MySQL** (soon!) - - **PostgreSQL** (soon!) +- ✔️ Multiple storage methods using [ass StorageEngines](https://github.com/tycrek/ass-storage-engine) (JSON by default) + - **File** + - **JSON** + - **YAML** (soon!) + - **Databases** + - **Mongo** (soon!) + - **MySQL** (soon!) + - **PostgreSQL** (soon!) ### Access types @@ -186,12 +190,41 @@ ass aims to support these storage methods at a minimum: - **JSON** - **Mongo** (soon) -- **MySQL** (soon) -- **PostgreSQL** (soon) An ass StorageEngine implements support for one type of database (or file, such as JSON or YAML). This lets ass server hosts pick their database of choice, because all they'll have to do is plugin the connection/authentication details, then ass will handle the rest, using the resource ID as the key. -The only storage engine ass comes with by default is **JSON**. Others will be published to [npm](https://www.npmjs.com/) and listed here. +The only storage engine ass comes with by default is **JSON**. Others will be published to [npm](https://www.npmjs.com/) and listed here. If you find (or create!) a StorageEngine you like, you can use it by installing it with `npm i ` then changing the contents of [`data.js`](https://github.com/tycrek/ass/blob/master/data.js). At this time, a modified `data.js` might look like this: + +```js +/** + * Used for global data management + */ + +//const { JsonStorageEngine } = require('@tycrek/ass-storage-engine'); +const { CustomStorageEngine } = require('my-custom-ass-storage-engine'); + +//const data = new JsonStorageEngine(); + +// StorageEngines may take no parameters... +const data1 = new CustomStorageEngine(); + +// multiple parameters... +const data2 = new CustomStorageEngine('Parameters!!', 420); + +// or object-based parameters, depending on what the StorageEngine dev decides on. +const data3 = new CustomStorageEngine({ key1: 'value1', key2: { key3: 44 } }); + +module.exports = data1; + +``` + +As long as the StorageEngine properly implements `GET`/`PUT`/`DEL`/`HAS` +StorageFunctions, replacing the file/database system is just that easy. + +If you develop & publish a Engine, feel free to [open a PR on this README](https://github.com/tycrek/ass/edit/master/README.md) to add it. + +- [`npm publish` docs](https://docs.npmjs.com/cli/v7/commands/npm-publish) +- ["How to publish packages to npm (the way the industry does things)"](https://zellwk.com/blog/publish-to-npm/)([`@tycrek/ass-storage-engine`](https://www.npmjs.com/package/@tycrek/ass-storage-engine) is published using the software this guide recommends, [np](https://github.com/sindresorhus/np)) **A wiki page on writing a custom StorageEngine is coming soon. Once complete, you can find it [here](https://github.com/tycrek/ass/wiki/Writing-a-StorageEngine).** From 130876c43ebf3c328395b1eabd1ed4d57a66be8c Mon Sep 17 00:00:00 2001 From: tycrek Date: Tue, 6 Jul 2021 14:50:47 -0600 Subject: [PATCH 7/7] DS --- routers/resource.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routers/resource.js b/routers/resource.js index f1b53b4..29c957e 100644 --- a/routers/resource.js +++ b/routers/resource.js @@ -18,7 +18,7 @@ router.use((req, res, next) => { // If the ID is invalid, return 404. Otherwise, continue normally data.has(req.ass.resourceId) - .then((has) => has ? next() : res.sendStatus(CODE_NOT_FOUND)) + .then((has) => has ? next() : res.sendStatus(CODE_NOT_FOUND)) // skipcq: JS-0229 .catch(next); }); @@ -98,7 +98,7 @@ router.get('/oembed', (req, res, next) => // Delete file router.get('/delete/:deleteId', (req, res, next) => { - let oldName, oldType; + let oldName, oldType; // skipcq: JS-0119 data.get(req.ass.resourceId) .then((fileData) => { // Extract info for logs @@ -115,7 +115,7 @@ router.get('/delete/:deleteId', (req, res, next) => { return Promise.all([s3enabled ? deleteS3(fileData) : fs.rmSync(path(fileData.path)), fs.rmSync(path(diskFilePath, 'thumbnails/', fileData.thumbnail))]); }) .then(() => data.del(req.ass.resourceId)) - .then(() => (log(`Deleted: ${oldName} (${oldType})`), res.type('text').send('File has been deleted!'))) + .then(() => (log(`Deleted: ${oldName} (${oldType})`), res.type('text').send('File has been deleted!'))) // skipcq: JS-0090 .catch(next); });