diff --git a/package.json b/package.json index 80522466b..a44b0f5ec 100644 --- a/package.json +++ b/package.json @@ -132,8 +132,8 @@ "semantic-release": "^17.4.4", "semantic-release-docker-buildx": "^1.0.1", "tailwindcss": "^2.2.2", - "ts-node": "^10.0.0", - "typescript": "^4.3.4" + "ts-node": "^10.2.1", + "typescript": "^4.4.3" }, "resolutions": { "sqlite3/node-gyp": "^5.1.0" diff --git a/server/api/plexapi.ts b/server/api/plexapi.ts index dea93c535..cee4a9cd1 100644 --- a/server/api/plexapi.ts +++ b/server/api/plexapi.ts @@ -1,5 +1,5 @@ import NodePlexAPI from 'plex-api'; -import { getSettings, PlexSettings } from '../lib/settings'; +import { getSettings, Library, PlexSettings } from '../lib/settings'; export interface PlexLibraryItem { ratingKey: string; @@ -11,11 +11,16 @@ export interface PlexLibraryItem { grandparentGuid?: string; addedAt: number; updatedAt: number; + Guid?: { + id: string; + }[]; type: 'movie' | 'show' | 'season' | 'episode'; + Media: Media[]; } interface PlexLibraryResponse { MediaContainer: { + totalSize: number; Metadata: PlexLibraryItem[]; }; } @@ -137,12 +142,50 @@ class PlexAPI { return response.MediaContainer.Directory; } - public async getLibraryContents(id: string): Promise { - const response = await this.plexClient.query( - `/library/sections/${id}/all` - ); + public async syncLibraries(): Promise { + const settings = getSettings(); + + const libraries = await this.getLibraries(); + + const newLibraries: Library[] = libraries + // Remove libraries that are not movie or show + .filter((library) => library.type === 'movie' || library.type === 'show') + // Remove libraries that do not have a metadata agent set (usually personal video libraries) + .filter((library) => library.agent !== 'com.plexapp.agents.none') + .map((library) => { + const existing = settings.plex.libraries.find( + (l) => l.id === library.key && l.name === library.title + ); + + return { + id: library.key, + name: library.title, + enabled: existing?.enabled ?? false, + type: library.type, + lastScan: existing?.lastScan, + }; + }); + + settings.plex.libraries = newLibraries; + settings.save(); + } + + public async getLibraryContents( + id: string, + { offset = 0, size = 50 }: { offset?: number; size?: number } = {} + ): Promise<{ totalSize: number; items: PlexLibraryItem[] }> { + const response = await this.plexClient.query({ + uri: `/library/sections/${id}/all?includeGuids=1`, + extraHeaders: { + 'X-Plex-Container-Start': `${offset}`, + 'X-Plex-Container-Size': `${size}`, + }, + }); - return response.MediaContainer.Metadata ?? []; + return { + totalSize: response.MediaContainer.totalSize, + items: response.MediaContainer.Metadata ?? [], + }; } public async getMetadata( @@ -166,10 +209,17 @@ class PlexAPI { return response.MediaContainer.Metadata; } - public async getRecentlyAdded(id: string): Promise { - const response = await this.plexClient.query( - `/library/sections/${id}/recentlyAdded` - ); + public async getRecentlyAdded( + id: string, + options: { addedAt: number } = { + addedAt: Date.now() - 1000 * 60 * 60, + } + ): Promise { + const response = await this.plexClient.query({ + uri: `/library/sections/${id}/all?sort=addedAt%3Adesc&addedAt>>=${Math.floor( + options.addedAt / 1000 + )}`, + }); return response.MediaContainer.Metadata; } diff --git a/server/index.ts b/server/index.ts index 8471926e7..f85b02752 100644 --- a/server/index.ts +++ b/server/index.ts @@ -10,7 +10,9 @@ import path from 'path'; import swaggerUi from 'swagger-ui-express'; import { createConnection, getRepository } from 'typeorm'; import YAML from 'yamljs'; +import PlexAPI from './api/plexapi'; import { Session } from './entity/Session'; +import { User } from './entity/User'; import { startJobs } from './job/schedule'; import notificationManager from './lib/notifications'; import DiscordAgent from './lib/notifications/agents/discord'; @@ -49,6 +51,26 @@ app // Load Settings const settings = getSettings().load(); + // Migrate library types + if ( + settings.plex.libraries.length > 1 && + !settings.plex.libraries[0].type + ) { + const userRepository = getRepository(User); + const admin = await userRepository.findOne({ + select: ['id', 'plexToken'], + order: { id: 'ASC' }, + }); + + if (admin) { + const plexapi = new PlexAPI({ plexToken: admin.plexToken }); + await plexapi.syncLibraries(); + logger.info('Migrating libraries to include media type', { + label: 'Settings', + }); + } + } + // Register Notification Agents notificationManager.registerAgents([ new DiscordAgent(), diff --git a/server/lib/cache.ts b/server/lib/cache.ts index 3aa18244d..fa03783c8 100644 --- a/server/lib/cache.ts +++ b/server/lib/cache.ts @@ -1,6 +1,12 @@ import NodeCache from 'node-cache'; -export type AvailableCacheIds = 'tmdb' | 'radarr' | 'sonarr' | 'rt' | 'github'; +export type AvailableCacheIds = + | 'tmdb' + | 'radarr' + | 'sonarr' + | 'rt' + | 'github' + | 'plexguid'; const DEFAULT_TTL = 300; const DEFAULT_CHECK_PERIOD = 120; @@ -48,6 +54,10 @@ class CacheManager { stdTtl: 21600, checkPeriod: 60 * 30, }), + plexguid: new Cache('plexguid', 'Plex GUID Cache', { + stdTtl: 86400 * 7, // 1 week cache + checkPeriod: 60 * 30, + }), }; public getCache(id: AvailableCacheIds): Cache { diff --git a/server/lib/scanners/baseScanner.ts b/server/lib/scanners/baseScanner.ts index a39279c97..f76ea92b0 100644 --- a/server/lib/scanners/baseScanner.ts +++ b/server/lib/scanners/baseScanner.ts @@ -55,6 +55,7 @@ class BaseScanner { private updateRate; protected progress = 0; protected items: T[] = []; + protected totalSize?: number = 0; protected scannerName: string; protected enable4kMovie = false; protected enable4kShow = false; @@ -609,6 +610,14 @@ class BaseScanner { ): void { logger[level](message, { label: this.scannerName, ...optional }); } + + get protectedUpdateRate(): number { + return this.updateRate; + } + + get protectedBundleSize(): number { + return this.bundleSize; + } } export default BaseScanner; diff --git a/server/lib/scanners/plex/index.ts b/server/lib/scanners/plex/index.ts index 27fcaa646..20835b9a6 100644 --- a/server/lib/scanners/plex/index.ts +++ b/server/lib/scanners/plex/index.ts @@ -4,6 +4,7 @@ import animeList from '../../../api/animelist'; import PlexAPI, { PlexLibraryItem, PlexMetadata } from '../../../api/plexapi'; import { TmdbTvDetails } from '../../../api/themoviedb/interfaces'; import { User } from '../../../entity/User'; +import cacheManager from '../../cache'; import { getSettings, Library } from '../../settings'; import BaseScanner, { MediaIds, @@ -38,7 +39,7 @@ class PlexScanner private isRecentOnly = false; public constructor(isRecentOnly = false) { - super('Plex Scan'); + super('Plex Scan', { bundleSize: 50 }); this.isRecentOnly = isRecentOnly; } @@ -46,7 +47,7 @@ class PlexScanner return { running: this.running, progress: this.progress, - total: this.items.length, + total: this.totalSize ?? 0, currentLibrary: this.currentLibrary, libraries: this.libraries, }; @@ -82,10 +83,17 @@ class PlexScanner this.currentLibrary = library; this.log( `Beginning to process recently added for library: ${library.name}`, - 'info' + 'info', + { lastScan: library.lastScan } ); const libraryItems = await this.plexClient.getRecentlyAdded( - library.id + library.id, + library.lastScan + ? { + // We remove 10 minutes from the last scan as a buffer + addedAt: library.lastScan - 1000 * 60 * 10, + } + : undefined ); // Bundle items up by rating keys @@ -104,13 +112,26 @@ class PlexScanner }); await this.loop(this.processItem.bind(this), { sessionId }); + + // After run completes, update last scan time + const newLibraries = settings.plex.libraries.map((lib) => { + if (lib.id === library.id) { + return { + ...lib, + lastScan: Date.now(), + }; + } + return lib; + }); + + settings.plex.libraries = newLibraries; + settings.save(); } } else { for (const library of this.libraries) { this.currentLibrary = library; this.log(`Beginning to process library: ${library.name}`, 'info'); - this.items = await this.plexClient.getLibraryContents(library.id); - await this.loop(this.processItem.bind(this), { sessionId }); + await this.paginateLibrary(library, { sessionId }); } } this.log( @@ -126,6 +147,52 @@ class PlexScanner } } + private async paginateLibrary( + library: Library, + { start = 0, sessionId }: { start?: number; sessionId: string } + ) { + if (!this.running) { + throw new Error('Sync was aborted.'); + } + + if (this.sessionId !== sessionId) { + throw new Error('New session was started. Old session aborted.'); + } + + const response = await this.plexClient.getLibraryContents(library.id, { + size: this.protectedBundleSize, + offset: start, + }); + + this.progress = start; + this.totalSize = response.totalSize; + + if (response.items.length === 0) { + return; + } + + await Promise.all( + response.items.map(async (item) => { + await this.processItem(item); + }) + ); + + if (response.items.length < this.protectedBundleSize) { + return; + } + + await new Promise((resolve, reject) => + setTimeout(() => { + this.paginateLibrary(library, { + start: start + this.protectedBundleSize, + sessionId, + }) + .then(() => resolve()) + .catch((e) => reject(new Error(e.message))); + }, this.protectedUpdateRate) + ); + } + private async processItem(plexitem: PlexLibraryItem) { try { if (plexitem.type === 'movie') { @@ -147,9 +214,8 @@ class PlexScanner private async processPlexMovie(plexitem: PlexLibraryItem) { const mediaIds = await this.getMediaIds(plexitem); - const metadata = await this.plexClient.getMetadata(plexitem.ratingKey); - const has4k = metadata.Media.some( + const has4k = plexitem.Media.some( (media) => media.videoResolution === '4k' ); @@ -263,10 +329,25 @@ class PlexScanner } private async getMediaIds(plexitem: PlexLibraryItem): Promise { - const mediaIds: Partial = {}; + let mediaIds: Partial = {}; // Check if item is using new plex movie/tv agent if (plexitem.guid.match(plexRegex)) { - const metadata = await this.plexClient.getMetadata(plexitem.ratingKey); + const guidCache = cacheManager.getCache('plexguid'); + + const cachedGuids = guidCache.data.get(plexitem.ratingKey); + + if (cachedGuids) { + this.log('GUIDs are cached. Skipping metadata request.', 'debug', { + mediaIds: cachedGuids, + title: plexitem.title, + }); + mediaIds = cachedGuids; + } + + const metadata = + plexitem.Guid && plexitem.Guid.length > 0 + ? plexitem + : await this.plexClient.getMetadata(plexitem.ratingKey); // If there is no Guid field at all, then we bail if (!metadata.Guid) { @@ -295,6 +376,10 @@ class PlexScanner }); mediaIds.tmdbId = tmdbMovie.id; } + + // Cache GUIDs + guidCache.data.set(plexitem.ratingKey, mediaIds); + // Check if the agent is IMDb } else if (plexitem.guid.match(imdbRegex)) { const imdbMatch = plexitem.guid.match(imdbRegex); diff --git a/server/lib/settings.ts b/server/lib/settings.ts index dc00f80cb..8ece986e9 100644 --- a/server/lib/settings.ts +++ b/server/lib/settings.ts @@ -9,6 +9,8 @@ export interface Library { id: string; name: string; enabled: boolean; + type: 'show' | 'movie'; + lastScan?: number; } export interface Region { diff --git a/server/routes/settings/index.ts b/server/routes/settings/index.ts index 514aa1de9..bf8cfcdcc 100644 --- a/server/routes/settings/index.ts +++ b/server/routes/settings/index.ts @@ -20,7 +20,7 @@ import { scheduledJobs } from '../../job/schedule'; import cacheManager, { AvailableCacheIds } from '../../lib/cache'; import { Permission } from '../../lib/permissions'; import { plexFullScanner } from '../../lib/scanners/plex'; -import { getSettings, Library, MainSettings } from '../../lib/settings'; +import { getSettings, MainSettings } from '../../lib/settings'; import logger from '../../logger'; import { isAuthenticated } from '../../middleware/auth'; import { getAppVersion } from '../../utils/appVersion'; @@ -197,26 +197,7 @@ settingsRoutes.get('/plex/library', async (req, res) => { }); const plexapi = new PlexAPI({ plexToken: admin.plexToken }); - const libraries = await plexapi.getLibraries(); - - const newLibraries: Library[] = libraries - // Remove libraries that are not movie or show - .filter((library) => library.type === 'movie' || library.type === 'show') - // Remove libraries that do not have a metadata agent set (usually personal video libraries) - .filter((library) => library.agent !== 'com.plexapp.agents.none') - .map((library) => { - const existing = settings.plex.libraries.find( - (l) => l.id === library.key && l.name === library.title - ); - - return { - id: library.key, - name: library.title, - enabled: existing?.enabled ?? false, - }; - }); - - settings.plex.libraries = newLibraries; + await plexapi.syncLibraries(); } const enabledLibraries = req.query.enable diff --git a/server/types/plex-api.d.ts b/server/types/plex-api.d.ts index 2e6cdc165..77be0ff49 100644 --- a/server/types/plex-api.d.ts +++ b/server/types/plex-api.d.ts @@ -21,6 +21,13 @@ declare module 'plex-api' { requestOptions?: Record; }); // eslint-disable-next-line @typescript-eslint/no-explicit-any - query: >(endpoint: string) => Promise; + query: >( + endpoint: + | string + | { + uri: string; + extraHeaders?: Record; + } + ) => Promise; } } diff --git a/tsconfig.json b/tsconfig.json index 5a10e8bfb..a8f3bb650 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -15,7 +15,8 @@ "jsx": "preserve", "strictPropertyInitialization": false, "experimentalDecorators": true, - "emitDecoratorMetadata": true + "emitDecoratorMetadata": true, + "useUnknownInCatchVariables": false }, "include": ["next-env.d.ts", "src/**/*.ts", "src/**/*.tsx"], "exclude": ["node_modules"] diff --git a/yarn.lock b/yarn.lock index c4cc96c31..93ee9c9e7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1247,6 +1247,18 @@ dependencies: chalk "^4.0.0" +"@cspotcode/source-map-consumer@0.8.0": + version "0.8.0" + resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b" + integrity sha512-41qniHzTU8yAGbCp04ohlmSrZf8bkf/iJsl3V0dRGsQN/5GFfx+LbCSsCpp2gqrqjTVg/K6O8ycoV35JIwAzAg== + +"@cspotcode/source-map-support@0.6.1": + version "0.6.1" + resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.6.1.tgz#118511f316e2e87ee4294761868e254d3da47960" + integrity sha512-DX3Z+T5dt1ockmPdobJS/FAsQPW4V4SrWEhD2iYQT2Cb2tQsiMnYxrcUH9By/Z3B+v0S5LMBkQtV/XOBbpLEOg== + dependencies: + "@cspotcode/source-map-consumer" "0.8.0" + "@dabh/diagnostics@^2.0.2": version "2.0.2" resolved "https://registry.yarnpkg.com/@dabh/diagnostics/-/diagnostics-2.0.2.tgz#290d08f7b381b8f94607dc8f471a12c675f9db31" @@ -2164,10 +2176,10 @@ resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.1.tgz#95f2d167ffb9b8d2068b0b235302fafd4df711f2" integrity sha512-509r2+yARFfHHE7T6Puu2jjkoycftovhXRqW328PDXTVGKihlb1P8Z9mMZH04ebyajfRY7dedfGynlrFHJUQCg== -"@tsconfig/node16@^1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.1.tgz#a6ca6a9a0ff366af433f42f5f0e124794ff6b8f1" - integrity sha512-FTgBI767POY/lKNDNbIzgAX6miIDBs6NTCbdlDb8TrWovHsSvaVIZDlTqym29C6UqhzwcJx4CYr+AlrMywA0cA== +"@tsconfig/node16@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.2.tgz#423c77877d0569db20e1fc80885ac4118314010e" + integrity sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA== "@types/babel__core@^7.1.7": version "7.1.9" @@ -2808,6 +2820,11 @@ acorn-walk@^7.0.0: resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc" integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA== +acorn-walk@^8.1.1: + version "8.2.0" + resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1" + integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== + acorn@^7.0.0, acorn@^7.1.1: version "7.4.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" @@ -2818,6 +2835,11 @@ acorn@^7.4.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.0.tgz#e1ad486e6c54501634c6c397c5c121daa383607c" integrity sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w== +acorn@^8.4.1: + version "8.5.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.5.0.tgz#4512ccb99b3698c752591e9bb4472e38ad43cee2" + integrity sha512-yXbYeFy+jUuYd3/CDcg2NkIYE991XYX/bje7LmjJigUciaeO1JR4XxXgCIV1/Zc/dRuFEyw1L0pbA+qynJkW5Q== + agent-base@4, agent-base@^4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.3.0.tgz#8165f01c436009bccad0b1d122f05ed770efc6ee" @@ -12695,14 +12717,6 @@ source-map-resolve@^0.5.0: source-map-url "^0.4.0" urix "^0.1.0" -source-map-support@^0.5.17: - version "0.5.19" - resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.19.tgz#a98b62f86dcaf4f67399648c085291ab9e8fed61" - integrity sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw== - dependencies: - buffer-from "^1.0.0" - source-map "^0.6.0" - source-map-url@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.0.tgz#3e935d7ddd73631b97659956d55128e87b5084a3" @@ -12730,7 +12744,7 @@ source-map@^0.5.0, source-map@^0.5.6, source-map@^0.5.7: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w= -source-map@^0.6.0, source-map@^0.6.1: +source-map@^0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== @@ -13651,20 +13665,22 @@ ts-easing@^0.2.0: resolved "https://registry.yarnpkg.com/ts-easing/-/ts-easing-0.2.0.tgz#c8a8a35025105566588d87dbda05dd7fbfa5a4ec" integrity sha512-Z86EW+fFFh/IFB1fqQ3/+7Zpf9t2ebOAxNI/V6Wo7r5gqiqtxmgTlQ1qbqQcjLKYeSHPTsEmvlJUDg/EuL0uHQ== -ts-node@^10.0.0: - version "10.0.0" - resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.0.0.tgz#05f10b9a716b0b624129ad44f0ea05dac84ba3be" - integrity sha512-ROWeOIUvfFbPZkoDis0L/55Fk+6gFQNZwwKPLinacRl6tsxstTF1DbAcLKkovwnpKMVvOMHP1TIbnwXwtLg1gg== +ts-node@^10.2.1: + version "10.2.1" + resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.2.1.tgz#4cc93bea0a7aba2179497e65bb08ddfc198b3ab5" + integrity sha512-hCnyOyuGmD5wHleOQX6NIjJtYVIO8bPP8F2acWkB4W06wdlkgyvJtubO/I9NkI88hCFECbsEgoLc0VNkYmcSfw== dependencies: + "@cspotcode/source-map-support" "0.6.1" "@tsconfig/node10" "^1.0.7" "@tsconfig/node12" "^1.0.7" "@tsconfig/node14" "^1.0.0" - "@tsconfig/node16" "^1.0.1" + "@tsconfig/node16" "^1.0.2" + acorn "^8.4.1" + acorn-walk "^8.1.1" arg "^4.1.0" create-require "^1.1.0" diff "^4.0.1" make-error "^1.1.1" - source-map-support "^0.5.17" yn "3.1.1" ts-pnp@^1.1.6: @@ -13828,10 +13844,10 @@ typescript@^4.0: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.0.2.tgz#7ea7c88777c723c681e33bf7988be5d008d05ac2" integrity sha512-e4ERvRV2wb+rRZ/IQeb3jm2VxBsirQLpQhdxplZ2MEzGvDkkMmPglecnNDfSUBivMjP93vRbngYYDQqQ/78bcQ== -typescript@^4.3.4: - version "4.3.4" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.3.4.tgz#3f85b986945bcf31071decdd96cf8bfa65f9dcbc" - integrity sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew== +typescript@^4.4.3: + version "4.4.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.4.3.tgz#bdc5407caa2b109efd4f82fe130656f977a29324" + integrity sha512-4xfscpisVgqqDfPaJo5vkd+Qd/ItkoagnHpufr+i2QCHBsNYp+G7UAoyFl8aPtx879u38wPV65rZ8qbGZijalA== uc.micro@^1.0.1: version "1.0.6"