open-source-mirrors
/
ass
mirror of https://github.com/tycrek/ass
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added support for S3 uploads (deletion NOT implemented yet)

Browse Source
pull/18/head
tycrek 3 years ago
parent f2e52c7e1b
commit af51401b7b
No known key found for this signature in database
GPG Key ID: 25D74F3943625263
5 changed files with 114 additions and 84 deletions
Show Stats Download Patch File Download Diff File

158
ass.js
Unescape View File

@ -7,13 +7,14 @@ try {
}
// Load the config
const { host, port, domain, useSsl, resourceIdSize, gfyIdSize, resourceIdType, isProxied, diskFilePath, saveWithDate, saveAsOriginal, s3enabled, s3bucket } = require('./config.json');
const { host, port, domain, useSsl, resourceIdSize, gfyIdSize, resourceIdType, isProxied, diskFilePath, saveWithDate, saveAsOriginal, s3enabled } = require('./config.json');
//#region Imports
const fs = require('fs-extra');
const express = require('express');
const useragent = require('express-useragent');
const rateLimit = require("express-rate-limit");
const fetch = require('node-fetch');
const marked = require('marked');
const multer = require('multer');
const DateTime = require('luxon').DateTime;
@ -22,7 +23,7 @@ const OpenGraph = require('./ogp');
const Thumbnail = require('./thumbnails');
const Vibrant = require('./vibrant');
const uploadS3 = require('./s3');
const { path, saveData, log, verify, generateToken, generateId, formatBytes, randomHexColour, arrayEquals } = require('./utils');
const { path, saveData, log, verify, generateToken, generateId, formatBytes, arrayEquals, getS3url, downloadTempS3 } = require('./utils');
//#endregion
//#region Variables, module setup
@ -110,10 +111,26 @@ function startup() {
max: 30 // Limit each IP to 30 requests per windowMs
}));
// Upload file
!s3enabled
? app.post('/', upload.single('file'), ({ next }) => next())
: app.post('/', (req, res, next) => uploadS3(req, res, (error) => (error ? console.error(error) : log(`File uploaded to S3 [${s3bucket}]`), next())));
// Upload file (local & S3)
s3enabled
? app.post('/', (req, res, next) => uploadS3(req, res, (error) => ((error) && console.error(error), next())))
: app.post('/', upload.single('file'), ({ next }) => next());
// Generate a thumbnail & get the Vibrant colour
app.post('/', (req, _res, next) => {
// Download a temp copy to work with if using S3 storage
(s3enabled ? downloadTempS3(req.file) : new Promise((resolve) => resolve()))
// Generate the thumbnail/vibrant
.then(() => Promise.all([Thumbnail(req.file), Vibrant(req.file)]))
.then(([thumbnail, vibrant]) => (req.file.thumbnail = thumbnail, req.file.vibrant = vibrant))
// Remove the temp file if using S3 storage
.then(() => s3enabled ? fs.remove(path('uploads/', req.file.originalname)) : null)
.then(() => next())
.catch((err) => next(err));
});
// Process uploaded file
app.post('/', (req, res) => {
@ -142,80 +159,79 @@ function startup() {
color: req.headers['x-ass-og-color']
};
// Generate a thumbnail & get the Vibrant colour
Promise.all([Thumbnail(req.file), (req.file.mimetype.includes('video') ? randomHexColour() : Vibrant(req.file))])
.then(([thumbnail, vibrant]) => (req.file.thumbnail = thumbnail, req.file.vibrant = vibrant))
.catch(console.error)
// Finish processing the file
.then(() => {
// Save the file information
let resourceId = generateId(generator, resourceIdSize, req.headers['x-ass-gfycat'] || gfyIdSize, req.file.originalname);
data[resourceId.split('.')[0]] = req.file;
saveData(data);
// Log the upload
let logInfo = `${req.file.originalname} (${req.file.mimetype})`;
log(`Uploaded: ${logInfo} (user: ${users[uploadToken] ? users[uploadToken].username : '<token-only>'})`);
// Build the URLs
let resourceUrl = `${getTrueHttp()}${trueDomain}/${resourceId}`;
let thumbnailUrl = `${getTrueHttp()}${trueDomain}/${resourceId}/thumbnail`;
let deleteUrl = `${getTrueHttp()}${trueDomain}/delete/${req.file.filename}`;
// Send the response
res.type('json').send({ resource: resourceUrl, thumbnail: thumbnailUrl, delete: deleteUrl })
.on('finish', () => {
// After we have sent the user the response, also send a Webhook to Discord (if headers are present)
if (req.headers['x-ass-webhook-client'] && req.headers['x-ass-webhook-token']) {
// Build the webhook client & embed
let whc = new WebhookClient(req.headers['x-ass-webhook-client'], req.headers['x-ass-webhook-token']);
let embed = new MessageEmbed()
.setTitle(logInfo)
.setURL(resourceUrl)
.setDescription(`**Size:** \`${formatBytes(req.file.size)}\`\n**[Delete](${deleteUrl})**`)
.setThumbnail(thumbnailUrl)
.setColor(req.file.vibrant)
.setTimestamp(req.file.timestamp);
// Send the embed to the webhook, then delete the client after to free resources
whc.send(null, {
username: req.headers['x-ass-webhook-username'] || 'ass',
avatarURL: req.headers['x-ass-webhook-avatar'] || ASS_LOGO,
embeds: [embed]
}).then((_msg) => whc.destroy());
}
// Also update the users upload count
if (!users[uploadToken]) {
let generator = () => generateId('random', 20, null);
let username = generator();
while (Object.values(users).findIndex((user) => user.username == username) != -1)
username = generator();
users[uploadToken] = { username, count: 0 };
}
users[uploadToken].count += 1;
fs.writeJsonSync(path('auth.json'), { users }, { spaces: 4 })
});
// Save the file information
let resourceId = generateId(generator, resourceIdSize, req.headers['x-ass-gfycat'] || gfyIdSize, req.file.originalname);
data[resourceId.split('.')[0]] = req.file;
saveData(data);
// Log the upload
let logInfo = `${req.file.originalname} (${req.file.mimetype})`;
log(`Uploaded: ${logInfo} (user: ${users[uploadToken] ? users[uploadToken].username : '<token-only>'})`);
// Build the URLs
let resourceUrl = `${getTrueHttp()}${trueDomain}/${resourceId}`;
let thumbnailUrl = `${getTrueHttp()}${trueDomain}/${resourceId}/thumbnail`;
let deleteUrl = `${getTrueHttp()}${trueDomain}/delete/${req.file.filename}`;
// Send the response
res.type('json').send({ resource: resourceUrl, thumbnail: thumbnailUrl, delete: deleteUrl })
.on('finish', () => {
// After we have sent the user the response, also send a Webhook to Discord (if headers are present)
if (req.headers['x-ass-webhook-client'] && req.headers['x-ass-webhook-token']) {
// Build the webhook client & embed
let whc = new WebhookClient(req.headers['x-ass-webhook-client'], req.headers['x-ass-webhook-token']);
let embed = new MessageEmbed()
.setTitle(logInfo)
.setURL(resourceUrl)
.setDescription(`**Size:** \`${formatBytes(req.file.size)}\`\n**[Delete](${deleteUrl})**`)
.setThumbnail(thumbnailUrl)
.setColor(req.file.vibrant)
.setTimestamp(req.file.timestamp);
// Send the embed to the webhook, then delete the client after to free resources
whc.send(null, {
username: req.headers['x-ass-webhook-username'] || 'ass',
avatarURL: req.headers['x-ass-webhook-avatar'] || ASS_LOGO,
embeds: [embed]
}).then((_msg) => whc.destroy());
}
// Also update the users upload count
if (!users[uploadToken]) {
let generator = () => generateId('random', 20, null);
let username = generator();
while (Object.values(users).findIndex((user) => user.username == username) != -1)
username = generator();
users[uploadToken] = { username, count: 0 };
}
users[uploadToken].count += 1;
fs.writeJsonSync(path('auth.json'), { users }, { spaces: 4 })
});
});
// View file
app.get('/:resourceId', (req, res) => {
let resourceId = req.ass.resourceId;
let fileData = data[resourceId];
// If the client is Discord, send an Open Graph embed
if (req.useragent.isBot) return res.type('html').send(new OpenGraph(getTrueHttp(), getTrueDomain(), resourceId, data[resourceId]).build());
if (req.useragent.isBot) return res.type('html').send(new OpenGraph(getTrueHttp(), getTrueDomain(), resourceId, fileData).build());
// Return the file differently depending on what storage option was used
let uploaders = {
s3: () => fetch(getS3url(fileData.originalname)).then((file) => {
file.headers.forEach((value, header) => res.setHeader(header, value));
file.body.pipe(res);
}),
local: () => {
res.header('Accept-Ranges', 'bytes').header('Content-Length', fileData.size).type(fileData.mimetype);
fs.createReadStream(path(fileData.path)).pipe(res);
}
};
// Read the file and send it to the client
fs.readFile(path(data[resourceId].path))
.then((fileData) => res
.header('Accept-Ranges', 'bytes')
.header('Content-Length', fileData.byteLength)
.type(data[resourceId].mimetype).send(fileData))
.catch(console.error);
uploaders[s3enabled ? 's3' : 'local']();
});
// Thumbnail response

7
ogp.js
Unescape View File

@ -2,7 +2,7 @@ const Mustache = require('mustache');
const DateTime = require('luxon').DateTime;
const { homepage, version } = require('./package.json');
const { s3enabled, s3endpoint, s3bucket } = require('./config.json');
const { formatBytes, randomHexColour } = require('./utils');
const { formatBytes, randomHexColour, getS3url } = require('./utils');
// https://ogp.me/
class OpenGraph {
@ -39,7 +39,7 @@ class OpenGraph {
}
build() {
let resourceUrl = !s3enabled ? (this.http + this.domain + "/" + this.resourceId + (this.type.includes('video') ? '.mp4' : this.type.includes('gif') ? '.gif' : '')) : `https://${s3bucket}.${s3endpoint}/${this.filename}`;
let resourceUrl = !s3enabled ? (this.http + this.domain + "/" + this.resourceId + (this.type.includes('video') ? '.mp4' : this.type.includes('gif') ? '.gif' : '')) : getS3url(this.filename);
return Mustache.render(html, {
homepage,
version,
@ -49,8 +49,7 @@ class OpenGraph {
resourceId: this.resourceId,
resourceUrl,
media: this.type.includes('video') ? `<video src="${resourceUrl}" style="height: 50vh;">` : `<img src="${resourceUrl}" style="height: 50vh;">`,
media: `<${this.type.includes('video') ? 'video' : 'img'} src="${resourceUrl}" style="height: 50vh;">`,
ogtype: this.type.includes('video') ? 'video.other' : 'image',
type: this.type.includes('video') ? 'video' : 'image',
title: (this.title.length != 0) ? `<meta property="og:title" content="${this.title}">` : '',

7
thumbnails.js
Unescape View File

@ -2,7 +2,8 @@ const ffmpeg = require('ffmpeg-static');
const Jimp = require('jimp');
const shell = require('any-shell-escape');
const { exec } = require('child_process');
const { path } = require('./utils');
const { path, getS3url } = require('./utils');
const { s3enabled } = require('./config.json');
const THUMBNAIL_QUALITY = 50;
const THUMBNAIL_SIZE = 512;
@ -19,12 +20,12 @@ function getCommand(src, dest) {
}
function getVideoThumbnail(file) {
return new Promise((resolve, reject) => exec(getCommand(path(file.path), getNewNamePath(file.originalname)), (err) => err ? reject(err) : resolve()));
return new Promise((resolve, reject) => exec(getCommand(s3enabled ? path('uploads/', file.originalname) : path(file.path), getNewNamePath(file.originalname)), (err) => err ? reject(err) : resolve()));
}
function getResizedThumbnail(file) {
return new Promise((resolve, reject) =>
Jimp.read(path(file.path))
Jimp.read(s3enabled ? getS3url(file.originalname) : path(file.path))
.then((image) => image
.quality(THUMBNAIL_QUALITY)
.resize(THUMBNAIL_SIZE, THUMBNAIL_SIZE, Jimp.RESIZE_BICUBIC)

13
utils.js
Unescape View File

@ -1,9 +1,11 @@
const fs = require('fs-extra');
const Path = require('path');
const fetch = require('node-fetch');
const token = require('./generators/token');
const zwsGen = require('./generators/zws');
const randomGen = require('./generators/random');
const gfyGen = require('./generators/gfycat');
const { s3bucket, s3endpoint } = require('./config.json');
const idModes = {
zws: 'zws', // Zero-width spaces (see: https://zws.im/)
@ -37,5 +39,14 @@ module.exports = {
colour += letters[(Math.floor(Math.random() * 16))];
return colour;
},
arrayEquals: (arr1, arr2) => arr1.length === arr2.length && arr1.slice().sort().every((value, index) => value === arr2.slice().sort()[index])
arrayEquals: (arr1, arr2) => arr1.length === arr2.length && arr1.slice().sort().every((value, index) => value === arr2.slice().sort()[index]),
downloadTempS3: (file) => new Promise((resolve, reject) =>
fetch(getS3url(file.originalname))
.then((f2) => f2.body.pipe(fs.createWriteStream(Path.join(__dirname, 'uploads/', file.originalname)).on('close', () => resolve())))
.catch(reject)),
getS3url,
}
function getS3url(originalName) {
return `https://${s3bucket}.${s3endpoint}/${originalName}`;
}

13
vibrant.js
Unescape View File

@ -1,12 +1,15 @@
const Vibrant = require('node-vibrant');
const { path } = require('./utils');
const { path, randomHexColour } = require('./utils');
const { s3enabled } = require('./config.json');
const COLOR_COUNT = 256;
const QUALITY = 3;
module.exports = (file) =>
new Promise((resolve, reject) =>
Vibrant.from(path(file.path))
.maxColorCount(COLOR_COUNT).quality(QUALITY).getPalette()
.then((palettes) => resolve(palettes[Object.keys(palettes).sort((a, b) => palettes[b].population - palettes[a].population)[0]].hex))
.catch(reject));
file.mimetype.includes('video')
? resolve(randomHexColour())
: Vibrant.from(s3enabled ? path('uploads/', file.originalname) : path(file.path))
.maxColorCount(COLOR_COUNT).quality(QUALITY).getPalette()
.then((palettes) => resolve(palettes[Object.keys(palettes).sort((a, b) => palettes[b].population - palettes[a].population)[0]].hex))
.catch(reject));

Write Preview
Loading…
Cancel
Save