diff --git a/_config.yml b/_config.yml deleted file mode 100644 index 2f7efbe..0000000 --- a/_config.yml +++ /dev/null @@ -1 +0,0 @@ -theme: jekyll-theme-minimal \ No newline at end of file diff --git a/ass.js b/ass.js index e99f5af..37308be 100755 --- a/ass.js +++ b/ass.js @@ -7,7 +7,7 @@ try { } // Load the config -const { host, port, domain, useSsl, resourceIdSize, gfyIdSize, resourceIdType, isProxied, diskFilePath, saveWithDate, saveAsOriginal, s3enabled } = require('./config.json'); +const { host, port, domain, useSsl, resourceIdSize, gfyIdSize, resourceIdType, isProxied, s3enabled, saveAsOriginal } = require('./config.json'); //#region Imports const fs = require('fs-extra'); @@ -16,13 +16,14 @@ const useragent = require('express-useragent'); const rateLimit = require("express-rate-limit"); const fetch = require('node-fetch'); const marked = require('marked'); -const multer = require('multer'); const DateTime = require('luxon').DateTime; const { WebhookClient, MessageEmbed } = require('discord.js'); const OpenGraph = require('./ogp'); const Thumbnail = require('./thumbnails'); const Vibrant = require('./vibrant'); -const uploadS3 = require('./s3'); +const Hash = require('./hash'); +const Path = require('path'); +const { uploadLocal, uploadS3 } = require('./storage'); const { path, saveData, log, verify, generateToken, generateId, formatBytes, arrayEquals, getS3url, downloadTempS3 } = require('./utils'); //#endregion @@ -31,23 +32,6 @@ const ASS_LOGO = 'https://cdn.discordapp.com/icons/848274994375294986/8d339d4a2f const app = express(); // Configure filename and location settings -const storage = multer.diskStorage({ - filename: saveAsOriginal ? (_req, file, callback) => callback(null, file.originalname) : null, - destination: !saveWithDate ? diskFilePath : (_req, _file, callback) => { - // Get current month and year - let [month, _day, year] = new Date().toLocaleDateString("en-US").split("/"); - - // Add 0 before single digit months eg ( 6 turns into 06) - let folder = `${diskFilePath}/${year}-${("0" + month).slice(-2)}`; - - // Create folder if it doesn't exist - fs.ensureDirSync(folder); - - callback(null, folder); - } -}); - -var upload = multer({ storage }); var users = {}; var data = {}; //#endregion @@ -90,6 +74,8 @@ function startup() { app.set('view engine', 'pug'); app.use(useragent.express()); + app.use((req, _res, next) => (req.randomId = generateId('random', 32, null, null), next())); + // Don't process favicon requests app.use((req, res, next) => req.url.includes('favicon.ico') ? res.sendStatus(204) : next()); @@ -114,22 +100,40 @@ function startup() { // Upload file (local & S3) s3enabled ? app.post('/', (req, res, next) => uploadS3(req, res, (error) => ((error) && console.error(error), next()))) - : app.post('/', upload.single('file'), ({ next }) => next()); + : app.post('/', uploadLocal, ({ next }) => next()); - // Generate a thumbnail & get the Vibrant colour + // Pre-response operations app.post('/', (req, _res, next) => { + req.file.randomId = req.randomId; // Download a temp copy to work with if using S3 storage (s3enabled ? downloadTempS3(req.file) : new Promise((resolve) => resolve())) - // Generate the thumbnail/vibrant - .then(() => Promise.all([Thumbnail(req.file), Vibrant(req.file)])) - .then(([thumbnail, vibrant]) => (req.file.thumbnail = thumbnail, req.file.vibrant = vibrant)) + // Generate the Thumbnail, Vibrant, and SHA1 hash + .then(() => Promise.all([Thumbnail(req.file), Vibrant(req.file), Hash(req.file)])) + .then(([thumbnail, vibrant, sha1]) => ( + req.file.thumbnail = thumbnail, + req.file.vibrant = vibrant, + req.file.sha1 = sha1 + )) - // Remove the temp file if using S3 storage - .then(() => s3enabled ? fs.remove(path('uploads/', req.file.originalname)) : null) + // Remove the temp file if using S3 storage, otherwise rename the local file + .then(() => s3enabled ? fs.remove(path('uploads/', req.file.originalname)) : renameFile(saveAsOriginal ? req.file.originalname : req.file.sha1)) .then(() => next()) .catch((err) => next(err)); + + function renameFile(newName) { + return new Promise((resolve, reject) => { + try { + let paths = [req.file.destination, newName]; + fs.rename(path(req.file.path), path(...paths)); + req.file.path = Path.join(...paths); + resolve(); + } catch (err) { + reject(err); + } + }); + } }); // Process uploaded file @@ -217,11 +221,11 @@ function startup() { let fileData = data[resourceId]; // If the client is Discord, send an Open Graph embed - if (req.useragent.isBot) return res.type('html').send(new OpenGraph(getTrueHttp(), getTrueDomain(), resourceId, fileData).build()); + if (req.useragent.isBot) return res.type('html').send(new OpenGraph(getTrueHttp(), getTrueDomain(), resourceId, fileData.randomId, fileData).build()); // Return the file differently depending on what storage option was used let uploaders = { - s3: () => fetch(getS3url(fileData.originalname)).then((file) => { + s3: () => fetch(getS3url(fileData.randomId)).then((file) => { file.headers.forEach((value, header) => res.setHeader(header, value)); file.body.pipe(res); }), diff --git a/hash.js b/hash.js new file mode 100644 index 0000000..40d128d --- /dev/null +++ b/hash.js @@ -0,0 +1,13 @@ +const fs = require('fs-extra'); +const crypto = require('crypto'); +const toArray = require('stream-to-array') +const { path } = require('./utils'); +const { s3enabled } = require('./config.json'); + +module.exports = (file) => + new Promise((resolve, reject) => + toArray((fs.createReadStream(s3enabled ? path('uploads/', file.originalname) : path(file.path)))) + .then((parts) => Buffer.concat(parts.map((part) => Buffer.isBuffer(part) ? part : Buffer.from(part)))) + .then((buf) => crypto.createHash('sha1').update(buf).digest('hex')) + .then(resolve) + .catch(reject)); diff --git a/ogp.js b/ogp.js index 795b4ed..ea8ea1d 100644 --- a/ogp.js +++ b/ogp.js @@ -1,7 +1,7 @@ const Mustache = require('mustache'); const DateTime = require('luxon').DateTime; const { homepage, version } = require('./package.json'); -const { s3enabled, s3endpoint, s3bucket } = require('./config.json'); +const { s3enabled } = require('./config.json'); const { formatBytes, randomHexColour, getS3url } = require('./utils'); // https://ogp.me/ @@ -9,6 +9,7 @@ class OpenGraph { http; domain; resourceId; + randomId; filename; type; @@ -21,10 +22,11 @@ class OpenGraph { author; color; - constructor(http, domain, resourceId, { originalname, mimetype, size, timestamp, opengraph, vibrant }) { + constructor(http, domain, resourceId, randomId, { originalname, mimetype, size, timestamp, opengraph, vibrant }) { this.http = http; this.domain = domain; this.resourceId = resourceId; + this.randomId = randomId; this.type = mimetype; this.filename = originalname; @@ -39,7 +41,7 @@ class OpenGraph { } build() { - let resourceUrl = !s3enabled ? (this.http + this.domain + "/" + this.resourceId + (this.type.includes('video') ? '.mp4' : this.type.includes('gif') ? '.gif' : '')) : getS3url(this.filename); + let resourceUrl = !s3enabled ? (this.http + this.domain + "/" + this.resourceId + (this.type.includes('video') ? '.mp4' : this.type.includes('gif') ? '.gif' : '')) : getS3url(this.randomId); return Mustache.render(html, { homepage, version, diff --git a/s3.js b/s3.js deleted file mode 100644 index 3b7d412..0000000 --- a/s3.js +++ /dev/null @@ -1,24 +0,0 @@ -// https://docs.digitalocean.com/products/spaces/resources/s3-sdk-examples/ -// https://www.digitalocean.com/community/tutorials/how-to-upload-a-file-to-object-storage-with-node-js - -const aws = require('aws-sdk'); -const multer = require('multer'); -const multerS3 = require('multer-s3'); -const { s3endpoint, s3bucket, s3accessKey, s3secretKey } = require('./config.json'); - -const s3 = new aws.S3({ - endpoint: new aws.Endpoint(s3endpoint), - credentials: new aws.Credentials({ accessKeyId: s3accessKey, secretAccessKey: s3secretKey }) -}); - -const upload = multer({ - storage: multerS3({ - s3: s3, - bucket: s3bucket, - acl: 'public-read', - key: (_req, file, cb) => cb(null, file.originalname), - contentType: (_req, file, cb) => cb(null, file.mimetype) - }) -}).single('file'); - -module.exports = upload; diff --git a/storage.js b/storage.js new file mode 100644 index 0000000..8addc20 --- /dev/null +++ b/storage.js @@ -0,0 +1,45 @@ +// https://docs.digitalocean.com/products/spaces/resources/s3-sdk-examples/ +// https://www.digitalocean.com/community/tutorials/how-to-upload-a-file-to-object-storage-with-node-js + +const fs = require('fs-extra'); +const aws = require('aws-sdk'); +const multer = require('multer'); +const multerS3 = require('multer-s3'); +const { diskFilePath, saveWithDate, s3endpoint, s3bucket, s3accessKey, s3secretKey } = require('./config.json'); + +const s3 = new aws.S3({ + endpoint: new aws.Endpoint(s3endpoint), + credentials: new aws.Credentials({ accessKeyId: s3accessKey, secretAccessKey: s3secretKey }) +}); + +const uploadS3 = multer({ + storage: multerS3({ + s3: s3, + bucket: s3bucket, + acl: 'public-read', + key: (req, _file, cb) => cb(null, req.randomId), + contentType: (_req, file, cb) => cb(null, file.mimetype) + }) +}).single('file'); + +const uploadLocal = multer({ + storage: multer.diskStorage({ + destination: !saveWithDate ? diskFilePath : (_req, _file, cb) => { + // Get current month and year + let [month, _day, year] = new Date().toLocaleDateString("en-US").split("/"); + + // Add 0 before single digit months eg ( 6 turns into 06) + let folder = `${diskFilePath}/${year}-${("0" + month).slice(-2)}`; + + // Create folder if it doesn't exist + fs.ensureDirSync(folder); + + cb(null, folder); + } + }) +}).single('file'); + +module.exports = { uploadLocal, uploadS3 }; + +// This deletes everything from the Bucket +//s3.listObjects({ Bucket: s3bucket }, (err, data) => err ? console.error(err) : data['Contents'].forEach((obj) => s3.deleteObject({ Bucket: s3bucket, Key: obj['Key'] }, (err, data) => err ? console.log(err) : console.log(data)))); diff --git a/thumbnails.js b/thumbnails.js index ec63e90..981035f 100644 --- a/thumbnails.js +++ b/thumbnails.js @@ -25,7 +25,7 @@ function getVideoThumbnail(file) { function getResizedThumbnail(file) { return new Promise((resolve, reject) => - Jimp.read(s3enabled ? getS3url(file.originalname) : path(file.path)) + Jimp.read(s3enabled ? getS3url(file.randomId) : path(file.path)) .then((image) => image .quality(THUMBNAIL_QUALITY) .resize(THUMBNAIL_SIZE, THUMBNAIL_SIZE, Jimp.RESIZE_BICUBIC) diff --git a/utils.js b/utils.js index 444f0d7..f112ebd 100755 --- a/utils.js +++ b/utils.js @@ -41,12 +41,12 @@ module.exports = { }, arrayEquals: (arr1, arr2) => arr1.length === arr2.length && arr1.slice().sort().every((value, index) => value === arr2.slice().sort()[index]), downloadTempS3: (file) => new Promise((resolve, reject) => - fetch(getS3url(file.originalname)) + fetch(getS3url(file.randomId)) .then((f2) => f2.body.pipe(fs.createWriteStream(Path.join(__dirname, 'uploads/', file.originalname)).on('close', () => resolve()))) .catch(reject)), getS3url, } -function getS3url(originalName) { - return `https://${s3bucket}.${s3endpoint}/${originalName}`; +function getS3url(s3key) { + return `https://${s3bucket}.${s3endpoint}/${s3key}`; }