feat: custom image proxy (#3056)
Ryan Cohen
2 years ago
committed by
GitHub
parent
bfe56c3470
commit
500cd1f872
@ -0,0 +1,268 @@
|
||||
import logger from '@server/logger';
|
||||
import axios from 'axios';
|
||||
import rateLimit, { type rateLimitOptions } from 'axios-rate-limit';
|
||||
import { createHash } from 'crypto';
|
||||
import { promises } from 'fs';
|
||||
import path, { join } from 'path';
|
||||
|
||||
type ImageResponse = {
|
||||
meta: {
|
||||
revalidateAfter: number;
|
||||
curRevalidate: number;
|
||||
isStale: boolean;
|
||||
etag: string;
|
||||
extension: string;
|
||||
cacheKey: string;
|
||||
cacheMiss: boolean;
|
||||
};
|
||||
imageBuffer: Buffer;
|
||||
};
|
||||
|
||||
class ImageProxy {
|
||||
public static async clearCache(key: string) {
|
||||
let deletedImages = 0;
|
||||
const cacheDirectory = path.join(
|
||||
__dirname,
|
||||
'../../config/cache/images/',
|
||||
key
|
||||
);
|
||||
|
||||
const files = await promises.readdir(cacheDirectory);
|
||||
|
||||
for (const file of files) {
|
||||
const filePath = path.join(cacheDirectory, file);
|
||||
const stat = await promises.lstat(filePath);
|
||||
|
||||
if (stat.isDirectory()) {
|
||||
const imageFiles = await promises.readdir(filePath);
|
||||
|
||||
for (const imageFile of imageFiles) {
|
||||
const [, expireAtSt] = imageFile.split('.');
|
||||
const expireAt = Number(expireAtSt);
|
||||
const now = Date.now();
|
||||
|
||||
if (now > expireAt) {
|
||||
await promises.rm(path.join(filePath, imageFile));
|
||||
deletedImages += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`Cleared ${deletedImages} stale image(s) from cache`, {
|
||||
label: 'Image Cache',
|
||||
});
|
||||
}
|
||||
|
||||
public static async getImageStats(
|
||||
key: string
|
||||
): Promise<{ size: number; imageCount: number }> {
|
||||
const cacheDirectory = path.join(
|
||||
__dirname,
|
||||
'../../config/cache/images/',
|
||||
key
|
||||
);
|
||||
|
||||
const imageTotalSize = await ImageProxy.getDirectorySize(cacheDirectory);
|
||||
const imageCount = await ImageProxy.getImageCount(cacheDirectory);
|
||||
|
||||
return {
|
||||
size: imageTotalSize,
|
||||
imageCount,
|
||||
};
|
||||
}
|
||||
|
||||
private static async getDirectorySize(dir: string): Promise<number> {
|
||||
const files = await promises.readdir(dir, {
|
||||
withFileTypes: true,
|
||||
});
|
||||
|
||||
const paths = files.map(async (file) => {
|
||||
const path = join(dir, file.name);
|
||||
|
||||
if (file.isDirectory()) return await ImageProxy.getDirectorySize(path);
|
||||
|
||||
if (file.isFile()) {
|
||||
const { size } = await promises.stat(path);
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
return 0;
|
||||
});
|
||||
|
||||
return (await Promise.all(paths))
|
||||
.flat(Infinity)
|
||||
.reduce((i, size) => i + size, 0);
|
||||
}
|
||||
|
||||
private static async getImageCount(dir: string) {
|
||||
const files = await promises.readdir(dir);
|
||||
|
||||
return files.length;
|
||||
}
|
||||
|
||||
private axios;
|
||||
private cacheVersion;
|
||||
private key;
|
||||
|
||||
constructor(
|
||||
key: string,
|
||||
baseUrl: string,
|
||||
options: {
|
||||
cacheVersion?: number;
|
||||
rateLimitOptions?: rateLimitOptions;
|
||||
} = {}
|
||||
) {
|
||||
this.cacheVersion = options.cacheVersion ?? 1;
|
||||
this.key = key;
|
||||
this.axios = axios.create({
|
||||
baseURL: baseUrl,
|
||||
});
|
||||
|
||||
if (options.rateLimitOptions) {
|
||||
this.axios = rateLimit(this.axios, options.rateLimitOptions);
|
||||
}
|
||||
}
|
||||
|
||||
public async getImage(path: string): Promise<ImageResponse> {
|
||||
const cacheKey = this.getCacheKey(path);
|
||||
|
||||
const imageResponse = await this.get(cacheKey);
|
||||
|
||||
if (!imageResponse) {
|
||||
const newImage = await this.set(path, cacheKey);
|
||||
|
||||
if (!newImage) {
|
||||
throw new Error('Failed to load image');
|
||||
}
|
||||
|
||||
return newImage;
|
||||
}
|
||||
|
||||
// If the image is stale, we will revalidate it in the background.
|
||||
if (imageResponse.meta.isStale) {
|
||||
this.set(path, cacheKey);
|
||||
}
|
||||
|
||||
return imageResponse;
|
||||
}
|
||||
|
||||
private async get(cacheKey: string): Promise<ImageResponse | null> {
|
||||
try {
|
||||
const directory = join(this.getCacheDirectory(), cacheKey);
|
||||
const files = await promises.readdir(directory);
|
||||
const now = Date.now();
|
||||
|
||||
for (const file of files) {
|
||||
const [maxAgeSt, expireAtSt, etag, extension] = file.split('.');
|
||||
const buffer = await promises.readFile(join(directory, file));
|
||||
const expireAt = Number(expireAtSt);
|
||||
const maxAge = Number(maxAgeSt);
|
||||
|
||||
return {
|
||||
meta: {
|
||||
curRevalidate: maxAge,
|
||||
revalidateAfter: maxAge * 1000 + now,
|
||||
isStale: now > expireAt,
|
||||
etag,
|
||||
extension,
|
||||
cacheKey,
|
||||
cacheMiss: false,
|
||||
},
|
||||
imageBuffer: buffer,
|
||||
};
|
||||
}
|
||||
} catch (e) {
|
||||
// No files. Treat as empty cache.
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private async set(
|
||||
path: string,
|
||||
cacheKey: string
|
||||
): Promise<ImageResponse | null> {
|
||||
try {
|
||||
const directory = join(this.getCacheDirectory(), cacheKey);
|
||||
const response = await this.axios.get(path, {
|
||||
responseType: 'arraybuffer',
|
||||
});
|
||||
|
||||
const buffer = Buffer.from(response.data, 'binary');
|
||||
const extension = path.split('.').pop() ?? '';
|
||||
const maxAge = Number(response.headers['cache-control'].split('=')[1]);
|
||||
const expireAt = Date.now() + maxAge * 1000;
|
||||
const etag = response.headers.etag.replace(/"/g, '');
|
||||
|
||||
await this.writeToCacheDir(
|
||||
directory,
|
||||
extension,
|
||||
maxAge,
|
||||
expireAt,
|
||||
buffer,
|
||||
etag
|
||||
);
|
||||
|
||||
return {
|
||||
meta: {
|
||||
curRevalidate: maxAge,
|
||||
revalidateAfter: expireAt,
|
||||
isStale: false,
|
||||
etag,
|
||||
extension,
|
||||
cacheKey,
|
||||
cacheMiss: true,
|
||||
},
|
||||
imageBuffer: buffer,
|
||||
};
|
||||
} catch (e) {
|
||||
logger.debug('Something went wrong caching image.', {
|
||||
label: 'Image Cache',
|
||||
errorMessage: e.message,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async writeToCacheDir(
|
||||
dir: string,
|
||||
extension: string,
|
||||
maxAge: number,
|
||||
expireAt: number,
|
||||
buffer: Buffer,
|
||||
etag: string
|
||||
) {
|
||||
const filename = join(dir, `${maxAge}.${expireAt}.${etag}.${extension}`);
|
||||
|
||||
await promises.rm(dir, { force: true, recursive: true }).catch(() => {
|
||||
// do nothing
|
||||
});
|
||||
|
||||
await promises.mkdir(dir, { recursive: true });
|
||||
await promises.writeFile(filename, buffer);
|
||||
}
|
||||
|
||||
private getCacheKey(path: string) {
|
||||
return this.getHash([this.key, this.cacheVersion, path]);
|
||||
}
|
||||
|
||||
private getHash(items: (string | number | Buffer)[]) {
|
||||
const hash = createHash('sha256');
|
||||
for (const item of items) {
|
||||
if (typeof item === 'number') hash.update(String(item));
|
||||
else {
|
||||
hash.update(item);
|
||||
}
|
||||
}
|
||||
// See https://en.wikipedia.org/wiki/Base64#Filenames
|
||||
return hash.digest('base64').replace(/\//g, '-');
|
||||
}
|
||||
|
||||
private getCacheDirectory() {
|
||||
return path.join(__dirname, '../../config/cache/images/', this.key);
|
||||
}
|
||||
}
|
||||
|
||||
export default ImageProxy;
|
||||
@ -0,0 +1,39 @@
|
||||
import ImageProxy from '@server/lib/imageproxy';
|
||||
import logger from '@server/logger';
|
||||
import { Router } from 'express';
|
||||
|
||||
const router = Router();
|
||||
const tmdbImageProxy = new ImageProxy('tmdb', 'https://image.tmdb.org', {
|
||||
rateLimitOptions: {
|
||||
maxRequests: 20,
|
||||
maxRPS: 50,
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Image Proxy
|
||||
*/
|
||||
router.get('/*', async (req, res) => {
|
||||
const imagePath = req.path.replace('/image', '');
|
||||
try {
|
||||
const imageData = await tmdbImageProxy.getImage(imagePath);
|
||||
|
||||
res.writeHead(200, {
|
||||
'Content-Type': `image/${imageData.meta.extension}`,
|
||||
'Content-Length': imageData.imageBuffer.length,
|
||||
'Cache-Control': `public, max-age=${imageData.meta.curRevalidate}`,
|
||||
'OS-Cache-Key': imageData.meta.cacheKey,
|
||||
'OS-Cache-Status': imageData.meta.cacheMiss ? 'MISS' : 'HIT',
|
||||
});
|
||||
|
||||
res.end(imageData.imageBuffer);
|
||||
} catch (e) {
|
||||
logger.error('Failed to proxy image', {
|
||||
imagePath,
|
||||
errorMessage: e.message,
|
||||
});
|
||||
res.status(500).send();
|
||||
}
|
||||
});
|
||||
|
||||
export default router;
|
||||
@ -1,18 +1,27 @@
|
||||
import useSettings from '@app/hooks/useSettings';
|
||||
import type { ImageProps } from 'next/image';
|
||||
import type { ImageLoader, ImageProps } from 'next/image';
|
||||
import Image from 'next/image';
|
||||
|
||||
const imageLoader: ImageLoader = ({ src }) => src;
|
||||
|
||||
/**
|
||||
* The CachedImage component should be used wherever
|
||||
* we want to offer the option to locally cache images.
|
||||
*
|
||||
* It uses the `next/image` Image component but overrides
|
||||
* the `unoptimized` prop based on the application setting `cacheImages`.
|
||||
**/
|
||||
const CachedImage = (props: ImageProps) => {
|
||||
const CachedImage = ({ src, ...props }: ImageProps) => {
|
||||
const { currentSettings } = useSettings();
|
||||
|
||||
return <Image unoptimized={!currentSettings.cacheImages} {...props} />;
|
||||
let imageUrl = src;
|
||||
|
||||
if (typeof imageUrl === 'string' && imageUrl.startsWith('http')) {
|
||||
const parsedUrl = new URL(imageUrl);
|
||||
|
||||
if (parsedUrl.host === 'image.tmdb.org' && currentSettings.cacheImages) {
|
||||
imageUrl = imageUrl.replace('https://image.tmdb.org', '/imageproxy');
|
||||
}
|
||||
}
|
||||
|
||||
return <Image unoptimized loader={imageLoader} src={imageUrl} {...props} />;
|
||||
};
|
||||
|
||||
export default CachedImage;
|
||||
|
||||
Loading…
Reference in new issue