feat: custom image proxy (#3056)

pull/3075/head
Ryan Cohen 1 year ago committed by GitHub
parent bfe56c3470
commit 500cd1f872
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

3
.gitignore vendored

@ -64,3 +64,6 @@ cypress/screenshots
# TS Build Info
tsconfig.tsbuildinfo
# Config Cache Directory
config/cache

@ -40,6 +40,14 @@ If you enable this setting and find yourself unable to access Overseerr, you can
This setting is **disabled** by default.
### Enable Image Caching
When enabled, Overseerr will proxy and cache images from pre-configured sources (such as TMDB). This can use a significant amount of disk space.
Images are saved in the `config/cache/images` and stale images are cleared out every 24 hours.
You should enable this if you are having issues with loading images directly from TMDB in your browser.
### Display Language
Set the default display language for Overseerr. Users can override this setting in their user settings.

@ -2475,29 +2475,44 @@ paths:
content:
application/json:
schema:
type: array
items:
type: object
properties:
id:
type: string
example: cache-id
name:
type: string
example: cache name
stats:
type: object
properties:
imageCache:
type: object
properties:
tmdb:
type: object
properties:
size:
type: number
example: 123456
imageCount:
type: number
example: 123
apiCaches:
type: array
items:
type: object
properties:
hits:
type: number
misses:
type: number
keys:
type: number
ksize:
type: number
vsize:
type: number
id:
type: string
example: cache-id
name:
type: string
example: cache name
stats:
type: object
properties:
hits:
type: number
misses:
type: number
keys:
type: number
ksize:
type: number
vsize:
type: number
/settings/cache/{cacheId}/flush:
post:
summary: Flush a specific cache

@ -17,6 +17,7 @@ import WebPushAgent from '@server/lib/notifications/agents/webpush';
import { getSettings } from '@server/lib/settings';
import logger from '@server/logger';
import routes from '@server/routes';
import imageproxy from '@server/routes/imageproxy';
import { getAppVersion } from '@server/utils/appVersion';
import restartFlag from '@server/utils/restartFlag';
import { getClientIp } from '@supercharge/request-ip';
@ -176,6 +177,9 @@ app
next();
});
server.use('/api/v1', routes);
server.use('/imageproxy', imageproxy);
server.get('*', (req, res) => handle(req, res));
server.use(
(

@ -51,6 +51,11 @@ export interface CacheItem {
};
}
export interface CacheResponse {
apiCaches: CacheItem[];
imageCache: Record<'tmdb', { size: number; imageCount: number }>;
}
export interface StatusResponse {
version: string;
commitTag: string;

@ -1,4 +1,5 @@
import downloadTracker from '@server/lib/downloadtracker';
import ImageProxy from '@server/lib/imageproxy';
import { plexFullScanner, plexRecentScanner } from '@server/lib/scanners/plex';
import { radarrScanner } from '@server/lib/scanners/radarr';
import { sonarrScanner } from '@server/lib/scanners/sonarr';
@ -133,5 +134,21 @@ export const startJobs = (): void => {
}),
});
// Run image cache cleanup every 5 minutes
scheduledJobs.push({
id: 'image-cache-cleanup',
name: 'Image Cache Cleanup',
type: 'process',
interval: 'long',
cronSchedule: jobs['image-cache-cleanup'].schedule,
job: schedule.scheduleJob(jobs['image-cache-cleanup'].schedule, () => {
logger.info('Starting scheduled job: Image Cache Cleanup', {
label: 'Jobs',
});
// Clean TMDB image cache
ImageProxy.clearCache('tmdb');
}),
});
logger.info('Scheduled jobs loaded', { label: 'Jobs' });
};

@ -0,0 +1,268 @@
import logger from '@server/logger';
import axios from 'axios';
import rateLimit, { type rateLimitOptions } from 'axios-rate-limit';
import { createHash } from 'crypto';
import { promises } from 'fs';
import path, { join } from 'path';
type ImageResponse = {
meta: {
revalidateAfter: number;
curRevalidate: number;
isStale: boolean;
etag: string;
extension: string;
cacheKey: string;
cacheMiss: boolean;
};
imageBuffer: Buffer;
};
class ImageProxy {
public static async clearCache(key: string) {
let deletedImages = 0;
const cacheDirectory = path.join(
__dirname,
'../../config/cache/images/',
key
);
const files = await promises.readdir(cacheDirectory);
for (const file of files) {
const filePath = path.join(cacheDirectory, file);
const stat = await promises.lstat(filePath);
if (stat.isDirectory()) {
const imageFiles = await promises.readdir(filePath);
for (const imageFile of imageFiles) {
const [, expireAtSt] = imageFile.split('.');
const expireAt = Number(expireAtSt);
const now = Date.now();
if (now > expireAt) {
await promises.rm(path.join(filePath, imageFile));
deletedImages += 1;
}
}
}
}
logger.info(`Cleared ${deletedImages} stale image(s) from cache`, {
label: 'Image Cache',
});
}
public static async getImageStats(
key: string
): Promise<{ size: number; imageCount: number }> {
const cacheDirectory = path.join(
__dirname,
'../../config/cache/images/',
key
);
const imageTotalSize = await ImageProxy.getDirectorySize(cacheDirectory);
const imageCount = await ImageProxy.getImageCount(cacheDirectory);
return {
size: imageTotalSize,
imageCount,
};
}
private static async getDirectorySize(dir: string): Promise<number> {
const files = await promises.readdir(dir, {
withFileTypes: true,
});
const paths = files.map(async (file) => {
const path = join(dir, file.name);
if (file.isDirectory()) return await ImageProxy.getDirectorySize(path);
if (file.isFile()) {
const { size } = await promises.stat(path);
return size;
}
return 0;
});
return (await Promise.all(paths))
.flat(Infinity)
.reduce((i, size) => i + size, 0);
}
private static async getImageCount(dir: string) {
const files = await promises.readdir(dir);
return files.length;
}
private axios;
private cacheVersion;
private key;
constructor(
key: string,
baseUrl: string,
options: {
cacheVersion?: number;
rateLimitOptions?: rateLimitOptions;
} = {}
) {
this.cacheVersion = options.cacheVersion ?? 1;
this.key = key;
this.axios = axios.create({
baseURL: baseUrl,
});
if (options.rateLimitOptions) {
this.axios = rateLimit(this.axios, options.rateLimitOptions);
}
}
public async getImage(path: string): Promise<ImageResponse> {
const cacheKey = this.getCacheKey(path);
const imageResponse = await this.get(cacheKey);
if (!imageResponse) {
const newImage = await this.set(path, cacheKey);
if (!newImage) {
throw new Error('Failed to load image');
}
return newImage;
}
// If the image is stale, we will revalidate it in the background.
if (imageResponse.meta.isStale) {
this.set(path, cacheKey);
}
return imageResponse;
}
private async get(cacheKey: string): Promise<ImageResponse | null> {
try {
const directory = join(this.getCacheDirectory(), cacheKey);
const files = await promises.readdir(directory);
const now = Date.now();
for (const file of files) {
const [maxAgeSt, expireAtSt, etag, extension] = file.split('.');
const buffer = await promises.readFile(join(directory, file));
const expireAt = Number(expireAtSt);
const maxAge = Number(maxAgeSt);
return {
meta: {
curRevalidate: maxAge,
revalidateAfter: maxAge * 1000 + now,
isStale: now > expireAt,
etag,
extension,
cacheKey,
cacheMiss: false,
},
imageBuffer: buffer,
};
}
} catch (e) {
// No files. Treat as empty cache.
}
return null;
}
private async set(
path: string,
cacheKey: string
): Promise<ImageResponse | null> {
try {
const directory = join(this.getCacheDirectory(), cacheKey);
const response = await this.axios.get(path, {
responseType: 'arraybuffer',
});
const buffer = Buffer.from(response.data, 'binary');
const extension = path.split('.').pop() ?? '';
const maxAge = Number(response.headers['cache-control'].split('=')[1]);
const expireAt = Date.now() + maxAge * 1000;
const etag = response.headers.etag.replace(/"/g, '');
await this.writeToCacheDir(
directory,
extension,
maxAge,
expireAt,
buffer,
etag
);
return {
meta: {
curRevalidate: maxAge,
revalidateAfter: expireAt,
isStale: false,
etag,
extension,
cacheKey,
cacheMiss: true,
},
imageBuffer: buffer,
};
} catch (e) {
logger.debug('Something went wrong caching image.', {
label: 'Image Cache',
errorMessage: e.message,
});
return null;
}
}
private async writeToCacheDir(
dir: string,
extension: string,
maxAge: number,
expireAt: number,
buffer: Buffer,
etag: string
) {
const filename = join(dir, `${maxAge}.${expireAt}.${etag}.${extension}`);
await promises.rm(dir, { force: true, recursive: true }).catch(() => {
// do nothing
});
await promises.mkdir(dir, { recursive: true });
await promises.writeFile(filename, buffer);
}
private getCacheKey(path: string) {
return this.getHash([this.key, this.cacheVersion, path]);
}
private getHash(items: (string | number | Buffer)[]) {
const hash = createHash('sha256');
for (const item of items) {
if (typeof item === 'number') hash.update(String(item));
else {
hash.update(item);
}
}
// See https://en.wikipedia.org/wiki/Base64#Filenames
return hash.digest('base64').replace(/\//g, '-');
}
private getCacheDirectory() {
return path.join(__dirname, '../../config/cache/images/', this.key);
}
}
export default ImageProxy;

@ -247,7 +247,8 @@ export type JobId =
| 'radarr-scan'
| 'sonarr-scan'
| 'download-sync'
| 'download-sync-reset';
| 'download-sync-reset'
| 'image-cache-cleanup';
interface AllSettings {
clientId: string;
@ -414,6 +415,9 @@ class Settings {
'download-sync-reset': {
schedule: '0 0 1 * * *',
},
'image-cache-cleanup': {
schedule: '0 0 5 * * *',
},
},
};
if (initialSettings) {

@ -0,0 +1,39 @@
import ImageProxy from '@server/lib/imageproxy';
import logger from '@server/logger';
import { Router } from 'express';
const router = Router();
const tmdbImageProxy = new ImageProxy('tmdb', 'https://image.tmdb.org', {
rateLimitOptions: {
maxRequests: 20,
maxRPS: 50,
},
});
/**
* Image Proxy
*/
router.get('/*', async (req, res) => {
const imagePath = req.path.replace('/image', '');
try {
const imageData = await tmdbImageProxy.getImage(imagePath);
res.writeHead(200, {
'Content-Type': `image/${imageData.meta.extension}`,
'Content-Length': imageData.imageBuffer.length,
'Cache-Control': `public, max-age=${imageData.meta.curRevalidate}`,
'OS-Cache-Key': imageData.meta.cacheKey,
'OS-Cache-Status': imageData.meta.cacheMiss ? 'MISS' : 'HIT',
});
res.end(imageData.imageBuffer);
} catch (e) {
logger.error('Failed to proxy image', {
imagePath,
errorMessage: e.message,
});
res.status(500).send();
}
});
export default router;

@ -14,9 +14,10 @@ import type {
import { scheduledJobs } from '@server/job/schedule';
import type { AvailableCacheIds } from '@server/lib/cache';
import cacheManager from '@server/lib/cache';
import ImageProxy from '@server/lib/imageproxy';
import { Permission } from '@server/lib/permissions';
import { plexFullScanner } from '@server/lib/scanners/plex';
import type { MainSettings } from '@server/lib/settings';
import type { JobId, MainSettings } from '@server/lib/settings';
import { getSettings } from '@server/lib/settings';
import logger from '@server/logger';
import { isAuthenticated } from '@server/middleware/auth';
@ -491,7 +492,7 @@ settingsRoutes.post<{ jobId: string }>('/jobs/:jobId/run', (req, res, next) => {
});
});
settingsRoutes.post<{ jobId: string }>(
settingsRoutes.post<{ jobId: JobId }>(
'/jobs/:jobId/cancel',
(req, res, next) => {
const scheduledJob = scheduledJobs.find(
@ -518,7 +519,7 @@ settingsRoutes.post<{ jobId: string }>(
}
);
settingsRoutes.post<{ jobId: string }>(
settingsRoutes.post<{ jobId: JobId }>(
'/jobs/:jobId/schedule',
(req, res, next) => {
const scheduledJob = scheduledJobs.find(
@ -553,16 +554,23 @@ settingsRoutes.post<{ jobId: string }>(
}
);
settingsRoutes.get('/cache', (req, res) => {
const caches = cacheManager.getAllCaches();
settingsRoutes.get('/cache', async (_req, res) => {
const cacheManagerCaches = cacheManager.getAllCaches();
return res.status(200).json(
Object.values(caches).map((cache) => ({
id: cache.id,
name: cache.name,
stats: cache.getStats(),
}))
);
const apiCaches = Object.values(cacheManagerCaches).map((cache) => ({
id: cache.id,
name: cache.name,
stats: cache.getStats(),
}));
const tmdbImageCache = await ImageProxy.getImageStats('tmdb');
return res.status(200).json({
apiCaches,
imageCache: {
tmdb: tmdbImageCache,
},
});
});
settingsRoutes.post<{ cacheId: AvailableCacheIds }>(

@ -1,18 +1,27 @@
import useSettings from '@app/hooks/useSettings';
import type { ImageProps } from 'next/image';
import type { ImageLoader, ImageProps } from 'next/image';
import Image from 'next/image';
const imageLoader: ImageLoader = ({ src }) => src;
/**
* The CachedImage component should be used wherever
* we want to offer the option to locally cache images.
*
* It uses the `next/image` Image component but overrides
* the `unoptimized` prop based on the application setting `cacheImages`.
**/
const CachedImage = (props: ImageProps) => {
const CachedImage = ({ src, ...props }: ImageProps) => {
const { currentSettings } = useSettings();
return <Image unoptimized={!currentSettings.cacheImages} {...props} />;
let imageUrl = src;
if (typeof imageUrl === 'string' && imageUrl.startsWith('http')) {
const parsedUrl = new URL(imageUrl);
if (parsedUrl.host === 'image.tmdb.org' && currentSettings.cacheImages) {
imageUrl = imageUrl.replace('https://image.tmdb.org', '/imageproxy');
}
}
return <Image unoptimized loader={imageLoader} src={imageUrl} {...props} />;
};
export default CachedImage;

@ -1,3 +1,4 @@
import CachedImage from '@app/components/Common/CachedImage';
import Link from 'next/link';
import { useState } from 'react';
@ -30,11 +31,15 @@ const CompanyCard = ({ image, url, name }: CompanyCardProps) => {
role="link"
tabIndex={0}
>
<img
src={image}
alt={name}
className="relative z-40 max-h-full max-w-full"
/>
<div className="relative h-full w-full">
<CachedImage
src={image}
alt={name}
className="relative z-40 h-full w-full"
layout="fill"
objectFit="contain"
/>
</div>
<div
className={`absolute bottom-0 left-0 right-0 z-0 h-12 rounded-b-xl bg-gradient-to-t ${
isHovered ? 'from-gray-800' : 'from-gray-900'

@ -1,6 +1,8 @@
import TitleCard from '@app/components/TitleCard';
import { ArrowCircleRightIcon } from '@heroicons/react/solid';
import Link from 'next/link';
import { useState } from 'react';
import { useInView } from 'react-intersection-observer';
import { defineMessages, useIntl } from 'react-intl';
const messages = defineMessages({
@ -15,6 +17,18 @@ interface ShowMoreCardProps {
const ShowMoreCard = ({ url, posters }: ShowMoreCardProps) => {
const intl = useIntl();
const [isHovered, setHovered] = useState(false);
const { ref, inView } = useInView({
triggerOnce: true,
});
if (!inView) {
return (
<div ref={ref}>
<TitleCard.Placeholder />
</div>
);
}
return (
<Link href={url}>
<a

@ -11,7 +11,10 @@ import { formatBytes } from '@app/utils/numberHelpers';
import { Transition } from '@headlessui/react';
import { PlayIcon, StopIcon, TrashIcon } from '@heroicons/react/outline';
import { PencilIcon } from '@heroicons/react/solid';
import type { CacheItem } from '@server/interfaces/api/settingsInterfaces';
import type {
CacheItem,
CacheResponse,
} from '@server/interfaces/api/settingsInterfaces';
import type { JobId } from '@server/lib/settings';
import axios from 'axios';
import cronstrue from 'cronstrue/i18n';
@ -54,6 +57,7 @@ const messages: { [messageName: string]: MessageDescriptor } = defineMessages({
'sonarr-scan': 'Sonarr Scan',
'download-sync': 'Download Sync',
'download-sync-reset': 'Download Sync Reset',
'image-cache-cleanup': 'Image Cache Cleanup',
editJobSchedule: 'Modify Job',
jobScheduleEditSaved: 'Job edited successfully!',
jobScheduleEditFailed: 'Something went wrong while saving the job.',
@ -63,6 +67,11 @@ const messages: { [messageName: string]: MessageDescriptor } = defineMessages({
'Every {jobScheduleHours, plural, one {hour} other {{jobScheduleHours} hours}}',
editJobScheduleSelectorMinutes:
'Every {jobScheduleMinutes, plural, one {minute} other {{jobScheduleMinutes} minutes}}',
imagecache: 'Image Cache',
imagecacheDescription:
'When enabled in settings, Overseerr will proxy and cache images from pre-configured external sources. Cached images are saved into your config folder. You can find the files in <code>{appDataPath}/cache/images</code>.',
imagecachecount: 'Images Cached',
imagecachesize: 'Total Cache Size',
});
interface Job {
@ -128,7 +137,8 @@ const SettingsJobs = () => {
} = useSWR<Job[]>('/api/v1/settings/jobs', {
refreshInterval: 5000,
});
const { data: cacheData, mutate: cacheRevalidate } = useSWR<CacheItem[]>(
const { data: appData } = useSWR('/api/v1/status/appdata');
const { data: cacheData, mutate: cacheRevalidate } = useSWR<CacheResponse>(
'/api/v1/settings/cache',
{
refreshInterval: 10000,
@ -430,7 +440,7 @@ const SettingsJobs = () => {
</tr>
</thead>
<Table.TBody>
{cacheData?.map((cache) => (
{cacheData?.apiCaches.map((cache) => (
<tr key={`cache-list-${cache.id}`}>
<Table.TD>{cache.name}</Table.TD>
<Table.TD>{intl.formatNumber(cache.stats.hits)}</Table.TD>
@ -449,6 +459,41 @@ const SettingsJobs = () => {
</Table.TBody>
</Table>
</div>
<div>
<h3 className="heading">{intl.formatMessage(messages.imagecache)}</h3>
<p className="description">
{intl.formatMessage(messages.imagecacheDescription, {
code: (msg: React.ReactNode) => (
<code className="bg-opacity-50">{msg}</code>
),
appDataPath: appData ? appData.appDataPath : '/app/config',
})}
</p>
</div>
<div className="section">
<Table>
<thead>
<tr>
<Table.TH>{intl.formatMessage(messages.cachename)}</Table.TH>
<Table.TH>
{intl.formatMessage(messages.imagecachecount)}
</Table.TH>
<Table.TH>{intl.formatMessage(messages.imagecachesize)}</Table.TH>
</tr>
</thead>
<Table.TBody>
<tr>
<Table.TD>The Movie Database (tmdb)</Table.TD>
<Table.TD>
{intl.formatNumber(cacheData?.imageCache.tmdb.imageCount ?? 0)}
</Table.TD>
<Table.TD>
{formatBytes(cacheData?.imageCache.tmdb.size ?? 0)}
</Table.TD>
</tr>
</Table.TBody>
</Table>
</div>
</>
);
};

@ -46,7 +46,7 @@ const messages = defineMessages({
'Do NOT enable this setting unless you understand what you are doing!',
cacheImages: 'Enable Image Caching',
cacheImagesTip:
'Cache and serve optimized images (requires a significant amount of disk space)',
'Cache externally sourced images (requires a significant amount of disk space)',
trustProxy: 'Enable Proxy Support',
trustProxyTip:
'Allow Overseerr to correctly register client IP addresses behind a proxy',

@ -644,6 +644,11 @@
"components.Settings.SettingsJobsCache.editJobScheduleSelectorHours": "Every {jobScheduleHours, plural, one {hour} other {{jobScheduleHours} hours}}",
"components.Settings.SettingsJobsCache.editJobScheduleSelectorMinutes": "Every {jobScheduleMinutes, plural, one {minute} other {{jobScheduleMinutes} minutes}}",
"components.Settings.SettingsJobsCache.flushcache": "Flush Cache",
"components.Settings.SettingsJobsCache.image-cache-cleanup": "Image Cache Cleanup",
"components.Settings.SettingsJobsCache.imagecache": "Image Cache",
"components.Settings.SettingsJobsCache.imagecacheDescription": "When enabled in settings, Overseerr will proxy and cache images from pre-configured external sources. Cached images are saved into your config folder. You can find the files in <code>{appDataPath}/cache/images</code>.",
"components.Settings.SettingsJobsCache.imagecachecount": "Images Cached",
"components.Settings.SettingsJobsCache.imagecachesize": "Total Cache Size",
"components.Settings.SettingsJobsCache.jobScheduleEditFailed": "Something went wrong while saving the job.",
"components.Settings.SettingsJobsCache.jobScheduleEditSaved": "Job edited successfully!",
"components.Settings.SettingsJobsCache.jobcancelled": "{jobname} canceled.",
@ -754,7 +759,7 @@
"components.Settings.applicationTitle": "Application Title",
"components.Settings.applicationurl": "Application URL",
"components.Settings.cacheImages": "Enable Image Caching",
"components.Settings.cacheImagesTip": "Cache and serve optimized images (requires a significant amount of disk space)",
"components.Settings.cacheImagesTip": "Cache externally sourced images (requires a significant amount of disk space)",
"components.Settings.cancelscan": "Cancel Scan",
"components.Settings.copied": "Copied API key to clipboard.",
"components.Settings.csrfProtection": "Enable CSRF Protection",

Loading…
Cancel
Save