significantly improved S3 block storage

_config.yml

@@ -1 +0,0 @@
-theme: jekyll-theme-minimal

ass.js

@@ -7,7 +7,7 @@ try {
 }
 
 // Load the config
-const { host, port, domain, useSsl, resourceIdSize, gfyIdSize, resourceIdType, isProxied, diskFilePath, saveWithDate, saveAsOriginal, s3enabled } = require('./config.json');
+const { host, port, domain, useSsl, resourceIdSize, gfyIdSize, resourceIdType, isProxied, s3enabled, saveAsOriginal } = require('./config.json');
 
 //#region Imports
 const fs = require('fs-extra');
@@ -16,13 +16,14 @@ const useragent = require('express-useragent');
 const rateLimit = require("express-rate-limit");
 const fetch = require('node-fetch');
 const marked = require('marked');
-const multer = require('multer');
 const DateTime = require('luxon').DateTime;
 const { WebhookClient, MessageEmbed } = require('discord.js');
 const OpenGraph = require('./ogp');
 const Thumbnail = require('./thumbnails');
 const Vibrant = require('./vibrant');
-const uploadS3 = require('./s3');
+const Hash = require('./hash');
+const Path = require('path');
+const { uploadLocal, uploadS3 } = require('./storage');
 const { path, saveData, log, verify, generateToken, generateId, formatBytes, arrayEquals, getS3url, downloadTempS3 } = require('./utils');
 //#endregion
 
@@ -31,23 +32,6 @@ const ASS_LOGO = 'https://cdn.discordapp.com/icons/848274994375294986/8d339d4a2f
 const app = express();
 
 // Configure filename and location settings
-const storage = multer.diskStorage({
-	filename: saveAsOriginal ? (_req, file, callback) => callback(null, file.originalname) : null,
-	destination: !saveWithDate ? diskFilePath : (_req, _file, callback) => {
-		// Get current month and year
-		let [month, _day, year] = new Date().toLocaleDateString("en-US").split("/");
-
-		// Add 0 before single digit months eg ( 6 turns into 06)
-		let folder = `${diskFilePath}/${year}-${("0" + month).slice(-2)}`;
-
-		// Create folder if it doesn't exist
-		fs.ensureDirSync(folder);
-
-		callback(null, folder);
-	}
-});
-
-var upload = multer({ storage });
 var users = {};
 var data = {};
 //#endregion
@@ -90,6 +74,8 @@ function startup() {
 	app.set('view engine', 'pug');
 	app.use(useragent.express());
 
+	app.use((req, _res, next) => (req.randomId = generateId('random', 32, null, null), next()));
+
 	// Don't process favicon requests
 	app.use((req, res, next) => req.url.includes('favicon.ico') ? res.sendStatus(204) : next());
 
@@ -114,22 +100,40 @@ function startup() {
 	// Upload file (local & S3)
 	s3enabled
 		? app.post('/', (req, res, next) => uploadS3(req, res, (error) => ((error) && console.error(error), next())))
-		: app.post('/', upload.single('file'), ({ next }) => next());
+		: app.post('/', uploadLocal, ({ next }) => next());
 
-	// Generate a thumbnail & get the Vibrant colour
+	// Pre-response operations
 	app.post('/', (req, _res, next) => {
+		req.file.randomId = req.randomId;
 
 		// Download a temp copy to work with if using S3 storage
 		(s3enabled ? downloadTempS3(req.file) : new Promise((resolve) => resolve()))
 
-			// Generate the thumbnail/vibrant
-			.then(() => Promise.all([Thumbnail(req.file), Vibrant(req.file)]))
-			.then(([thumbnail, vibrant]) => (req.file.thumbnail = thumbnail, req.file.vibrant = vibrant))
+			// Generate the Thumbnail, Vibrant, and SHA1 hash
+			.then(() => Promise.all([Thumbnail(req.file), Vibrant(req.file), Hash(req.file)]))
+			.then(([thumbnail, vibrant, sha1]) => (
+				req.file.thumbnail = thumbnail,
+				req.file.vibrant = vibrant,
+				req.file.sha1 = sha1
+			))
 
-			// Remove the temp file if using S3 storage
-			.then(() => s3enabled ? fs.remove(path('uploads/', req.file.originalname)) : null)
+			// Remove the temp file if using S3 storage, otherwise rename the local file
+			.then(() => s3enabled ? fs.remove(path('uploads/', req.file.originalname)) : renameFile(saveAsOriginal ? req.file.originalname : req.file.sha1))
 			.then(() => next())
 			.catch((err) => next(err));
+
+		function renameFile(newName) {
+			return new Promise((resolve, reject) => {
+				try {
+					let paths = [req.file.destination, newName];
+					fs.rename(path(req.file.path), path(...paths));
+					req.file.path = Path.join(...paths);
+					resolve();
+				} catch (err) {
+					reject(err);
+				}
+			});
+		}
 	});
 
 	// Process uploaded file
@@ -217,11 +221,11 @@ function startup() {
 		let fileData = data[resourceId];
 
 		// If the client is Discord, send an Open Graph embed
-		if (req.useragent.isBot) return res.type('html').send(new OpenGraph(getTrueHttp(), getTrueDomain(), resourceId, fileData).build());
+		if (req.useragent.isBot) return res.type('html').send(new OpenGraph(getTrueHttp(), getTrueDomain(), resourceId, fileData.randomId, fileData).build());
 
 		// Return the file differently depending on what storage option was used
 		let uploaders = {
-			s3: () => fetch(getS3url(fileData.originalname)).then((file) => {
+			s3: () => fetch(getS3url(fileData.randomId)).then((file) => {
 				file.headers.forEach((value, header) => res.setHeader(header, value));
 				file.body.pipe(res);
 			}),

hash.js

@@ -0,0 +1,13 @@
+const fs = require('fs-extra');
+const crypto = require('crypto');
+const toArray = require('stream-to-array')
+const { path } = require('./utils');
+const { s3enabled } = require('./config.json');
+
+module.exports = (file) =>
+	new Promise((resolve, reject) =>
+		toArray((fs.createReadStream(s3enabled ? path('uploads/', file.originalname) : path(file.path))))
+			.then((parts) => Buffer.concat(parts.map((part) => Buffer.isBuffer(part) ? part : Buffer.from(part))))
+			.then((buf) => crypto.createHash('sha1').update(buf).digest('hex'))
+			.then(resolve)
+			.catch(reject));

ogp.js

@@ -1,7 +1,7 @@
 const Mustache = require('mustache');
 const DateTime = require('luxon').DateTime;
 const { homepage, version } = require('./package.json');
-const { s3enabled, s3endpoint, s3bucket } = require('./config.json');
+const { s3enabled } = require('./config.json');
 const { formatBytes, randomHexColour, getS3url } = require('./utils');
 
 // https://ogp.me/
@@ -9,6 +9,7 @@ class OpenGraph {
 	http;
 	domain;
 	resourceId;
+	randomId;
 
 	filename;
 	type;
@@ -21,10 +22,11 @@ class OpenGraph {
 	author;
 	color;
 
-	constructor(http, domain, resourceId, { originalname, mimetype, size, timestamp, opengraph, vibrant }) {
+	constructor(http, domain, resourceId, randomId, { originalname, mimetype, size, timestamp, opengraph, vibrant }) {
 		this.http = http;
 		this.domain = domain;
 		this.resourceId = resourceId;
+		this.randomId = randomId;
 
 		this.type = mimetype;
 		this.filename = originalname;
@@ -39,7 +41,7 @@ class OpenGraph {
 	}
 
 	build() {
-		let resourceUrl = !s3enabled ? (this.http + this.domain + "/" + this.resourceId + (this.type.includes('video') ? '.mp4' : this.type.includes('gif') ? '.gif' : '')) : getS3url(this.filename);
+		let resourceUrl = !s3enabled ? (this.http + this.domain + "/" + this.resourceId + (this.type.includes('video') ? '.mp4' : this.type.includes('gif') ? '.gif' : '')) : getS3url(this.randomId);
 		return Mustache.render(html, {
 			homepage,
 			version,

s3.js

@@ -1,24 +0,0 @@
-// https://docs.digitalocean.com/products/spaces/resources/s3-sdk-examples/
-// https://www.digitalocean.com/community/tutorials/how-to-upload-a-file-to-object-storage-with-node-js
-
-const aws = require('aws-sdk');
-const multer = require('multer');
-const multerS3 = require('multer-s3');
-const { s3endpoint, s3bucket, s3accessKey, s3secretKey } = require('./config.json');
-
-const s3 = new aws.S3({
-	endpoint: new aws.Endpoint(s3endpoint),
-	credentials: new aws.Credentials({ accessKeyId: s3accessKey, secretAccessKey: s3secretKey })
-});
-
-const upload = multer({
-	storage: multerS3({
-		s3: s3,
-		bucket: s3bucket,
-		acl: 'public-read',
-		key: (_req, file, cb) => cb(null, file.originalname),
-		contentType: (_req, file, cb) => cb(null, file.mimetype)
-	})
-}).single('file');
-
-module.exports = upload;

storage.js

@@ -0,0 +1,45 @@
+// https://docs.digitalocean.com/products/spaces/resources/s3-sdk-examples/
+// https://www.digitalocean.com/community/tutorials/how-to-upload-a-file-to-object-storage-with-node-js
+
+const fs = require('fs-extra');
+const aws = require('aws-sdk');
+const multer = require('multer');
+const multerS3 = require('multer-s3');
+const { diskFilePath, saveWithDate, s3endpoint, s3bucket, s3accessKey, s3secretKey } = require('./config.json');
+
+const s3 = new aws.S3({
+	endpoint: new aws.Endpoint(s3endpoint),
+	credentials: new aws.Credentials({ accessKeyId: s3accessKey, secretAccessKey: s3secretKey })
+});
+
+const uploadS3 = multer({
+	storage: multerS3({
+		s3: s3,
+		bucket: s3bucket,
+		acl: 'public-read',
+		key: (req, _file, cb) => cb(null, req.randomId),
+		contentType: (_req, file, cb) => cb(null, file.mimetype)
+	})
+}).single('file');
+
+const uploadLocal = multer({
+	storage: multer.diskStorage({
+		destination: !saveWithDate ? diskFilePath : (_req, _file, cb) => {
+			// Get current month and year
+			let [month, _day, year] = new Date().toLocaleDateString("en-US").split("/");
+
+			// Add 0 before single digit months eg ( 6 turns into 06)
+			let folder = `${diskFilePath}/${year}-${("0" + month).slice(-2)}`;
+
+			// Create folder if it doesn't exist
+			fs.ensureDirSync(folder);
+
+			cb(null, folder);
+		}
+	})
+}).single('file');
+
+module.exports = { uploadLocal, uploadS3 };
+
+// This deletes everything from the Bucket
+//s3.listObjects({ Bucket: s3bucket }, (err, data) => err ? console.error(err) : data['Contents'].forEach((obj) => s3.deleteObject({ Bucket: s3bucket, Key: obj['Key'] }, (err, data) => err ? console.log(err) : console.log(data))));

thumbnails.js

@@ -25,7 +25,7 @@ function getVideoThumbnail(file) {
 
 function getResizedThumbnail(file) {
 	return new Promise((resolve, reject) =>
-		Jimp.read(s3enabled ? getS3url(file.originalname) : path(file.path))
+		Jimp.read(s3enabled ? getS3url(file.randomId) : path(file.path))
 			.then((image) => image
 				.quality(THUMBNAIL_QUALITY)
 				.resize(THUMBNAIL_SIZE, THUMBNAIL_SIZE, Jimp.RESIZE_BICUBIC)

utils.js

@@ -41,12 +41,12 @@ module.exports = {
 	},
 	arrayEquals: (arr1, arr2) => arr1.length === arr2.length && arr1.slice().sort().every((value, index) => value === arr2.slice().sort()[index]),
 	downloadTempS3: (file) => new Promise((resolve, reject) =>
-		fetch(getS3url(file.originalname))
+		fetch(getS3url(file.randomId))
 			.then((f2) => f2.body.pipe(fs.createWriteStream(Path.join(__dirname, 'uploads/', file.originalname)).on('close', () => resolve())))
 			.catch(reject)),
 	getS3url,
 }
 
-function getS3url(originalName) {
-	return `https://${s3bucket}.${s3endpoint}/${originalName}`;
+function getS3url(s3key) {
+	return `https://${s3bucket}.${s3endpoint}/${s3key}`;
 }