From 500cd1f872942923d2b9c3b835e6329e335d4a3f Mon Sep 17 00:00:00 2001 From: Ryan Cohen Date: Tue, 18 Oct 2022 14:40:24 +0900 Subject: [PATCH] feat: custom image proxy (#3056) --- .gitignore | 3 + docs/using-overseerr/settings/README.md | 8 + overseerr-api.yml | 57 ++-- server/index.ts | 4 + server/interfaces/api/settingsInterfaces.ts | 5 + server/job/schedule.ts | 17 ++ server/lib/imageproxy.ts | 268 ++++++++++++++++++ server/lib/settings.ts | 6 +- server/routes/imageproxy.ts | 39 +++ server/routes/settings/index.ts | 32 ++- src/components/Common/CachedImage/index.tsx | 21 +- src/components/CompanyCard/index.tsx | 15 +- .../MediaSlider/ShowMoreCard/index.tsx | 14 + .../Settings/SettingsJobsCache/index.tsx | 51 +++- src/components/Settings/SettingsMain.tsx | 2 +- src/i18n/locale/en.json | 7 +- 16 files changed, 499 insertions(+), 50 deletions(-) create mode 100644 server/lib/imageproxy.ts create mode 100644 server/routes/imageproxy.ts diff --git a/.gitignore b/.gitignore index 4f7c3ce6..6e051447 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,6 @@ cypress/screenshots # TS Build Info tsconfig.tsbuildinfo + +# Config Cache Directory +config/cache diff --git a/docs/using-overseerr/settings/README.md b/docs/using-overseerr/settings/README.md index 82043073..477129fc 100644 --- a/docs/using-overseerr/settings/README.md +++ b/docs/using-overseerr/settings/README.md @@ -40,6 +40,14 @@ If you enable this setting and find yourself unable to access Overseerr, you can This setting is **disabled** by default. +### Enable Image Caching + +When enabled, Overseerr will proxy and cache images from pre-configured sources (such as TMDB). This can use a significant amount of disk space. + +Images are saved in the `config/cache/images` and stale images are cleared out every 24 hours. + +You should enable this if you are having issues with loading images directly from TMDB in your browser. + ### Display Language Set the default display language for Overseerr. Users can override this setting in their user settings. diff --git a/overseerr-api.yml b/overseerr-api.yml index 0be47e66..f114cce1 100644 --- a/overseerr-api.yml +++ b/overseerr-api.yml @@ -2475,29 +2475,44 @@ paths: content: application/json: schema: - type: array - items: - type: object - properties: - id: - type: string - example: cache-id - name: - type: string - example: cache name - stats: + type: object + properties: + imageCache: + type: object + properties: + tmdb: + type: object + properties: + size: + type: number + example: 123456 + imageCount: + type: number + example: 123 + apiCaches: + type: array + items: type: object properties: - hits: - type: number - misses: - type: number - keys: - type: number - ksize: - type: number - vsize: - type: number + id: + type: string + example: cache-id + name: + type: string + example: cache name + stats: + type: object + properties: + hits: + type: number + misses: + type: number + keys: + type: number + ksize: + type: number + vsize: + type: number /settings/cache/{cacheId}/flush: post: summary: Flush a specific cache diff --git a/server/index.ts b/server/index.ts index e78d5a57..12df2f1f 100644 --- a/server/index.ts +++ b/server/index.ts @@ -17,6 +17,7 @@ import WebPushAgent from '@server/lib/notifications/agents/webpush'; import { getSettings } from '@server/lib/settings'; import logger from '@server/logger'; import routes from '@server/routes'; +import imageproxy from '@server/routes/imageproxy'; import { getAppVersion } from '@server/utils/appVersion'; import restartFlag from '@server/utils/restartFlag'; import { getClientIp } from '@supercharge/request-ip'; @@ -176,6 +177,9 @@ app next(); }); server.use('/api/v1', routes); + + server.use('/imageproxy', imageproxy); + server.get('*', (req, res) => handle(req, res)); server.use( ( diff --git a/server/interfaces/api/settingsInterfaces.ts b/server/interfaces/api/settingsInterfaces.ts index 0e5ab45a..0cd2f171 100644 --- a/server/interfaces/api/settingsInterfaces.ts +++ b/server/interfaces/api/settingsInterfaces.ts @@ -51,6 +51,11 @@ export interface CacheItem { }; } +export interface CacheResponse { + apiCaches: CacheItem[]; + imageCache: Record<'tmdb', { size: number; imageCount: number }>; +} + export interface StatusResponse { version: string; commitTag: string; diff --git a/server/job/schedule.ts b/server/job/schedule.ts index ab606449..725e67b5 100644 --- a/server/job/schedule.ts +++ b/server/job/schedule.ts @@ -1,4 +1,5 @@ import downloadTracker from '@server/lib/downloadtracker'; +import ImageProxy from '@server/lib/imageproxy'; import { plexFullScanner, plexRecentScanner } from '@server/lib/scanners/plex'; import { radarrScanner } from '@server/lib/scanners/radarr'; import { sonarrScanner } from '@server/lib/scanners/sonarr'; @@ -133,5 +134,21 @@ export const startJobs = (): void => { }), }); + // Run image cache cleanup every 5 minutes + scheduledJobs.push({ + id: 'image-cache-cleanup', + name: 'Image Cache Cleanup', + type: 'process', + interval: 'long', + cronSchedule: jobs['image-cache-cleanup'].schedule, + job: schedule.scheduleJob(jobs['image-cache-cleanup'].schedule, () => { + logger.info('Starting scheduled job: Image Cache Cleanup', { + label: 'Jobs', + }); + // Clean TMDB image cache + ImageProxy.clearCache('tmdb'); + }), + }); + logger.info('Scheduled jobs loaded', { label: 'Jobs' }); }; diff --git a/server/lib/imageproxy.ts b/server/lib/imageproxy.ts new file mode 100644 index 00000000..34a097d5 --- /dev/null +++ b/server/lib/imageproxy.ts @@ -0,0 +1,268 @@ +import logger from '@server/logger'; +import axios from 'axios'; +import rateLimit, { type rateLimitOptions } from 'axios-rate-limit'; +import { createHash } from 'crypto'; +import { promises } from 'fs'; +import path, { join } from 'path'; + +type ImageResponse = { + meta: { + revalidateAfter: number; + curRevalidate: number; + isStale: boolean; + etag: string; + extension: string; + cacheKey: string; + cacheMiss: boolean; + }; + imageBuffer: Buffer; +}; + +class ImageProxy { + public static async clearCache(key: string) { + let deletedImages = 0; + const cacheDirectory = path.join( + __dirname, + '../../config/cache/images/', + key + ); + + const files = await promises.readdir(cacheDirectory); + + for (const file of files) { + const filePath = path.join(cacheDirectory, file); + const stat = await promises.lstat(filePath); + + if (stat.isDirectory()) { + const imageFiles = await promises.readdir(filePath); + + for (const imageFile of imageFiles) { + const [, expireAtSt] = imageFile.split('.'); + const expireAt = Number(expireAtSt); + const now = Date.now(); + + if (now > expireAt) { + await promises.rm(path.join(filePath, imageFile)); + deletedImages += 1; + } + } + } + } + + logger.info(`Cleared ${deletedImages} stale image(s) from cache`, { + label: 'Image Cache', + }); + } + + public static async getImageStats( + key: string + ): Promise<{ size: number; imageCount: number }> { + const cacheDirectory = path.join( + __dirname, + '../../config/cache/images/', + key + ); + + const imageTotalSize = await ImageProxy.getDirectorySize(cacheDirectory); + const imageCount = await ImageProxy.getImageCount(cacheDirectory); + + return { + size: imageTotalSize, + imageCount, + }; + } + + private static async getDirectorySize(dir: string): Promise { + const files = await promises.readdir(dir, { + withFileTypes: true, + }); + + const paths = files.map(async (file) => { + const path = join(dir, file.name); + + if (file.isDirectory()) return await ImageProxy.getDirectorySize(path); + + if (file.isFile()) { + const { size } = await promises.stat(path); + + return size; + } + + return 0; + }); + + return (await Promise.all(paths)) + .flat(Infinity) + .reduce((i, size) => i + size, 0); + } + + private static async getImageCount(dir: string) { + const files = await promises.readdir(dir); + + return files.length; + } + + private axios; + private cacheVersion; + private key; + + constructor( + key: string, + baseUrl: string, + options: { + cacheVersion?: number; + rateLimitOptions?: rateLimitOptions; + } = {} + ) { + this.cacheVersion = options.cacheVersion ?? 1; + this.key = key; + this.axios = axios.create({ + baseURL: baseUrl, + }); + + if (options.rateLimitOptions) { + this.axios = rateLimit(this.axios, options.rateLimitOptions); + } + } + + public async getImage(path: string): Promise { + const cacheKey = this.getCacheKey(path); + + const imageResponse = await this.get(cacheKey); + + if (!imageResponse) { + const newImage = await this.set(path, cacheKey); + + if (!newImage) { + throw new Error('Failed to load image'); + } + + return newImage; + } + + // If the image is stale, we will revalidate it in the background. + if (imageResponse.meta.isStale) { + this.set(path, cacheKey); + } + + return imageResponse; + } + + private async get(cacheKey: string): Promise { + try { + const directory = join(this.getCacheDirectory(), cacheKey); + const files = await promises.readdir(directory); + const now = Date.now(); + + for (const file of files) { + const [maxAgeSt, expireAtSt, etag, extension] = file.split('.'); + const buffer = await promises.readFile(join(directory, file)); + const expireAt = Number(expireAtSt); + const maxAge = Number(maxAgeSt); + + return { + meta: { + curRevalidate: maxAge, + revalidateAfter: maxAge * 1000 + now, + isStale: now > expireAt, + etag, + extension, + cacheKey, + cacheMiss: false, + }, + imageBuffer: buffer, + }; + } + } catch (e) { + // No files. Treat as empty cache. + } + + return null; + } + + private async set( + path: string, + cacheKey: string + ): Promise { + try { + const directory = join(this.getCacheDirectory(), cacheKey); + const response = await this.axios.get(path, { + responseType: 'arraybuffer', + }); + + const buffer = Buffer.from(response.data, 'binary'); + const extension = path.split('.').pop() ?? ''; + const maxAge = Number(response.headers['cache-control'].split('=')[1]); + const expireAt = Date.now() + maxAge * 1000; + const etag = response.headers.etag.replace(/"/g, ''); + + await this.writeToCacheDir( + directory, + extension, + maxAge, + expireAt, + buffer, + etag + ); + + return { + meta: { + curRevalidate: maxAge, + revalidateAfter: expireAt, + isStale: false, + etag, + extension, + cacheKey, + cacheMiss: true, + }, + imageBuffer: buffer, + }; + } catch (e) { + logger.debug('Something went wrong caching image.', { + label: 'Image Cache', + errorMessage: e.message, + }); + return null; + } + } + + private async writeToCacheDir( + dir: string, + extension: string, + maxAge: number, + expireAt: number, + buffer: Buffer, + etag: string + ) { + const filename = join(dir, `${maxAge}.${expireAt}.${etag}.${extension}`); + + await promises.rm(dir, { force: true, recursive: true }).catch(() => { + // do nothing + }); + + await promises.mkdir(dir, { recursive: true }); + await promises.writeFile(filename, buffer); + } + + private getCacheKey(path: string) { + return this.getHash([this.key, this.cacheVersion, path]); + } + + private getHash(items: (string | number | Buffer)[]) { + const hash = createHash('sha256'); + for (const item of items) { + if (typeof item === 'number') hash.update(String(item)); + else { + hash.update(item); + } + } + // See https://en.wikipedia.org/wiki/Base64#Filenames + return hash.digest('base64').replace(/\//g, '-'); + } + + private getCacheDirectory() { + return path.join(__dirname, '../../config/cache/images/', this.key); + } +} + +export default ImageProxy; diff --git a/server/lib/settings.ts b/server/lib/settings.ts index 5a2d2b8a..cf475554 100644 --- a/server/lib/settings.ts +++ b/server/lib/settings.ts @@ -247,7 +247,8 @@ export type JobId = | 'radarr-scan' | 'sonarr-scan' | 'download-sync' - | 'download-sync-reset'; + | 'download-sync-reset' + | 'image-cache-cleanup'; interface AllSettings { clientId: string; @@ -414,6 +415,9 @@ class Settings { 'download-sync-reset': { schedule: '0 0 1 * * *', }, + 'image-cache-cleanup': { + schedule: '0 0 5 * * *', + }, }, }; if (initialSettings) { diff --git a/server/routes/imageproxy.ts b/server/routes/imageproxy.ts new file mode 100644 index 00000000..6cf104f5 --- /dev/null +++ b/server/routes/imageproxy.ts @@ -0,0 +1,39 @@ +import ImageProxy from '@server/lib/imageproxy'; +import logger from '@server/logger'; +import { Router } from 'express'; + +const router = Router(); +const tmdbImageProxy = new ImageProxy('tmdb', 'https://image.tmdb.org', { + rateLimitOptions: { + maxRequests: 20, + maxRPS: 50, + }, +}); + +/** + * Image Proxy + */ +router.get('/*', async (req, res) => { + const imagePath = req.path.replace('/image', ''); + try { + const imageData = await tmdbImageProxy.getImage(imagePath); + + res.writeHead(200, { + 'Content-Type': `image/${imageData.meta.extension}`, + 'Content-Length': imageData.imageBuffer.length, + 'Cache-Control': `public, max-age=${imageData.meta.curRevalidate}`, + 'OS-Cache-Key': imageData.meta.cacheKey, + 'OS-Cache-Status': imageData.meta.cacheMiss ? 'MISS' : 'HIT', + }); + + res.end(imageData.imageBuffer); + } catch (e) { + logger.error('Failed to proxy image', { + imagePath, + errorMessage: e.message, + }); + res.status(500).send(); + } +}); + +export default router; diff --git a/server/routes/settings/index.ts b/server/routes/settings/index.ts index 41d2c745..7205b189 100644 --- a/server/routes/settings/index.ts +++ b/server/routes/settings/index.ts @@ -14,9 +14,10 @@ import type { import { scheduledJobs } from '@server/job/schedule'; import type { AvailableCacheIds } from '@server/lib/cache'; import cacheManager from '@server/lib/cache'; +import ImageProxy from '@server/lib/imageproxy'; import { Permission } from '@server/lib/permissions'; import { plexFullScanner } from '@server/lib/scanners/plex'; -import type { MainSettings } from '@server/lib/settings'; +import type { JobId, MainSettings } from '@server/lib/settings'; import { getSettings } from '@server/lib/settings'; import logger from '@server/logger'; import { isAuthenticated } from '@server/middleware/auth'; @@ -491,7 +492,7 @@ settingsRoutes.post<{ jobId: string }>('/jobs/:jobId/run', (req, res, next) => { }); }); -settingsRoutes.post<{ jobId: string }>( +settingsRoutes.post<{ jobId: JobId }>( '/jobs/:jobId/cancel', (req, res, next) => { const scheduledJob = scheduledJobs.find( @@ -518,7 +519,7 @@ settingsRoutes.post<{ jobId: string }>( } ); -settingsRoutes.post<{ jobId: string }>( +settingsRoutes.post<{ jobId: JobId }>( '/jobs/:jobId/schedule', (req, res, next) => { const scheduledJob = scheduledJobs.find( @@ -553,16 +554,23 @@ settingsRoutes.post<{ jobId: string }>( } ); -settingsRoutes.get('/cache', (req, res) => { - const caches = cacheManager.getAllCaches(); +settingsRoutes.get('/cache', async (_req, res) => { + const cacheManagerCaches = cacheManager.getAllCaches(); - return res.status(200).json( - Object.values(caches).map((cache) => ({ - id: cache.id, - name: cache.name, - stats: cache.getStats(), - })) - ); + const apiCaches = Object.values(cacheManagerCaches).map((cache) => ({ + id: cache.id, + name: cache.name, + stats: cache.getStats(), + })); + + const tmdbImageCache = await ImageProxy.getImageStats('tmdb'); + + return res.status(200).json({ + apiCaches, + imageCache: { + tmdb: tmdbImageCache, + }, + }); }); settingsRoutes.post<{ cacheId: AvailableCacheIds }>( diff --git a/src/components/Common/CachedImage/index.tsx b/src/components/Common/CachedImage/index.tsx index b1695937..6dfb8ee7 100644 --- a/src/components/Common/CachedImage/index.tsx +++ b/src/components/Common/CachedImage/index.tsx @@ -1,18 +1,27 @@ import useSettings from '@app/hooks/useSettings'; -import type { ImageProps } from 'next/image'; +import type { ImageLoader, ImageProps } from 'next/image'; import Image from 'next/image'; +const imageLoader: ImageLoader = ({ src }) => src; + /** * The CachedImage component should be used wherever * we want to offer the option to locally cache images. - * - * It uses the `next/image` Image component but overrides - * the `unoptimized` prop based on the application setting `cacheImages`. **/ -const CachedImage = (props: ImageProps) => { +const CachedImage = ({ src, ...props }: ImageProps) => { const { currentSettings } = useSettings(); - return ; + let imageUrl = src; + + if (typeof imageUrl === 'string' && imageUrl.startsWith('http')) { + const parsedUrl = new URL(imageUrl); + + if (parsedUrl.host === 'image.tmdb.org' && currentSettings.cacheImages) { + imageUrl = imageUrl.replace('https://image.tmdb.org', '/imageproxy'); + } + } + + return ; }; export default CachedImage; diff --git a/src/components/CompanyCard/index.tsx b/src/components/CompanyCard/index.tsx index 762d1a08..13b92a33 100644 --- a/src/components/CompanyCard/index.tsx +++ b/src/components/CompanyCard/index.tsx @@ -1,3 +1,4 @@ +import CachedImage from '@app/components/Common/CachedImage'; import Link from 'next/link'; import { useState } from 'react'; @@ -30,11 +31,15 @@ const CompanyCard = ({ image, url, name }: CompanyCardProps) => { role="link" tabIndex={0} > - {name} +
+ +
{ const intl = useIntl(); const [isHovered, setHovered] = useState(false); + const { ref, inView } = useInView({ + triggerOnce: true, + }); + + if (!inView) { + return ( +
+ +
+ ); + } + return ( {appDataPath}/cache/images.', + imagecachecount: 'Images Cached', + imagecachesize: 'Total Cache Size', }); interface Job { @@ -128,7 +137,8 @@ const SettingsJobs = () => { } = useSWR('/api/v1/settings/jobs', { refreshInterval: 5000, }); - const { data: cacheData, mutate: cacheRevalidate } = useSWR( + const { data: appData } = useSWR('/api/v1/status/appdata'); + const { data: cacheData, mutate: cacheRevalidate } = useSWR( '/api/v1/settings/cache', { refreshInterval: 10000, @@ -430,7 +440,7 @@ const SettingsJobs = () => { - {cacheData?.map((cache) => ( + {cacheData?.apiCaches.map((cache) => ( {cache.name} {intl.formatNumber(cache.stats.hits)} @@ -449,6 +459,41 @@ const SettingsJobs = () => {
+
+

{intl.formatMessage(messages.imagecache)}

+

+ {intl.formatMessage(messages.imagecacheDescription, { + code: (msg: React.ReactNode) => ( + {msg} + ), + appDataPath: appData ? appData.appDataPath : '/app/config', + })} +

+
+
+ + + + {intl.formatMessage(messages.cachename)} + + {intl.formatMessage(messages.imagecachecount)} + + {intl.formatMessage(messages.imagecachesize)} + + + + + The Movie Database (tmdb) + + {intl.formatNumber(cacheData?.imageCache.tmdb.imageCount ?? 0)} + + + {formatBytes(cacheData?.imageCache.tmdb.size ?? 0)} + + + +
+
); }; diff --git a/src/components/Settings/SettingsMain.tsx b/src/components/Settings/SettingsMain.tsx index ef0810f5..7d4e188e 100644 --- a/src/components/Settings/SettingsMain.tsx +++ b/src/components/Settings/SettingsMain.tsx @@ -46,7 +46,7 @@ const messages = defineMessages({ 'Do NOT enable this setting unless you understand what you are doing!', cacheImages: 'Enable Image Caching', cacheImagesTip: - 'Cache and serve optimized images (requires a significant amount of disk space)', + 'Cache externally sourced images (requires a significant amount of disk space)', trustProxy: 'Enable Proxy Support', trustProxyTip: 'Allow Overseerr to correctly register client IP addresses behind a proxy', diff --git a/src/i18n/locale/en.json b/src/i18n/locale/en.json index 2d8a64ba..d05928cd 100644 --- a/src/i18n/locale/en.json +++ b/src/i18n/locale/en.json @@ -644,6 +644,11 @@ "components.Settings.SettingsJobsCache.editJobScheduleSelectorHours": "Every {jobScheduleHours, plural, one {hour} other {{jobScheduleHours} hours}}", "components.Settings.SettingsJobsCache.editJobScheduleSelectorMinutes": "Every {jobScheduleMinutes, plural, one {minute} other {{jobScheduleMinutes} minutes}}", "components.Settings.SettingsJobsCache.flushcache": "Flush Cache", + "components.Settings.SettingsJobsCache.image-cache-cleanup": "Image Cache Cleanup", + "components.Settings.SettingsJobsCache.imagecache": "Image Cache", + "components.Settings.SettingsJobsCache.imagecacheDescription": "When enabled in settings, Overseerr will proxy and cache images from pre-configured external sources. Cached images are saved into your config folder. You can find the files in {appDataPath}/cache/images.", + "components.Settings.SettingsJobsCache.imagecachecount": "Images Cached", + "components.Settings.SettingsJobsCache.imagecachesize": "Total Cache Size", "components.Settings.SettingsJobsCache.jobScheduleEditFailed": "Something went wrong while saving the job.", "components.Settings.SettingsJobsCache.jobScheduleEditSaved": "Job edited successfully!", "components.Settings.SettingsJobsCache.jobcancelled": "{jobname} canceled.", @@ -754,7 +759,7 @@ "components.Settings.applicationTitle": "Application Title", "components.Settings.applicationurl": "Application URL", "components.Settings.cacheImages": "Enable Image Caching", - "components.Settings.cacheImagesTip": "Cache and serve optimized images (requires a significant amount of disk space)", + "components.Settings.cacheImagesTip": "Cache externally sourced images (requires a significant amount of disk space)", "components.Settings.cancelscan": "Cancel Scan", "components.Settings.copied": "Copied API key to clipboard.", "components.Settings.csrfProtection": "Enable CSRF Protection",