feat(plex-scan): plex scanner improvements (#2105)

pull/2106/head
Ryan Cohen 3 years ago committed by GitHub
parent 4c6009bc2c
commit afda9c7dc2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -132,8 +132,8 @@
"semantic-release": "^17.4.4",
"semantic-release-docker-buildx": "^1.0.1",
"tailwindcss": "^2.2.2",
"ts-node": "^10.0.0",
"typescript": "^4.3.4"
"ts-node": "^10.2.1",
"typescript": "^4.4.3"
},
"resolutions": {
"sqlite3/node-gyp": "^5.1.0"

@ -1,5 +1,5 @@
import NodePlexAPI from 'plex-api';
import { getSettings, PlexSettings } from '../lib/settings';
import { getSettings, Library, PlexSettings } from '../lib/settings';
export interface PlexLibraryItem {
ratingKey: string;
@ -11,11 +11,16 @@ export interface PlexLibraryItem {
grandparentGuid?: string;
addedAt: number;
updatedAt: number;
Guid?: {
id: string;
}[];
type: 'movie' | 'show' | 'season' | 'episode';
Media: Media[];
}
interface PlexLibraryResponse {
MediaContainer: {
totalSize: number;
Metadata: PlexLibraryItem[];
};
}
@ -137,12 +142,50 @@ class PlexAPI {
return response.MediaContainer.Directory;
}
public async getLibraryContents(id: string): Promise<PlexLibraryItem[]> {
const response = await this.plexClient.query<PlexLibraryResponse>(
`/library/sections/${id}/all`
);
public async syncLibraries(): Promise<void> {
const settings = getSettings();
const libraries = await this.getLibraries();
const newLibraries: Library[] = libraries
// Remove libraries that are not movie or show
.filter((library) => library.type === 'movie' || library.type === 'show')
// Remove libraries that do not have a metadata agent set (usually personal video libraries)
.filter((library) => library.agent !== 'com.plexapp.agents.none')
.map((library) => {
const existing = settings.plex.libraries.find(
(l) => l.id === library.key && l.name === library.title
);
return {
id: library.key,
name: library.title,
enabled: existing?.enabled ?? false,
type: library.type,
lastScan: existing?.lastScan,
};
});
settings.plex.libraries = newLibraries;
settings.save();
}
public async getLibraryContents(
id: string,
{ offset = 0, size = 50 }: { offset?: number; size?: number } = {}
): Promise<{ totalSize: number; items: PlexLibraryItem[] }> {
const response = await this.plexClient.query<PlexLibraryResponse>({
uri: `/library/sections/${id}/all?includeGuids=1`,
extraHeaders: {
'X-Plex-Container-Start': `${offset}`,
'X-Plex-Container-Size': `${size}`,
},
});
return response.MediaContainer.Metadata ?? [];
return {
totalSize: response.MediaContainer.totalSize,
items: response.MediaContainer.Metadata ?? [],
};
}
public async getMetadata(
@ -166,10 +209,17 @@ class PlexAPI {
return response.MediaContainer.Metadata;
}
public async getRecentlyAdded(id: string): Promise<PlexLibraryItem[]> {
const response = await this.plexClient.query<PlexLibraryResponse>(
`/library/sections/${id}/recentlyAdded`
);
public async getRecentlyAdded(
id: string,
options: { addedAt: number } = {
addedAt: Date.now() - 1000 * 60 * 60,
}
): Promise<PlexLibraryItem[]> {
const response = await this.plexClient.query<PlexLibraryResponse>({
uri: `/library/sections/${id}/all?sort=addedAt%3Adesc&addedAt>>=${Math.floor(
options.addedAt / 1000
)}`,
});
return response.MediaContainer.Metadata;
}

@ -10,7 +10,9 @@ import path from 'path';
import swaggerUi from 'swagger-ui-express';
import { createConnection, getRepository } from 'typeorm';
import YAML from 'yamljs';
import PlexAPI from './api/plexapi';
import { Session } from './entity/Session';
import { User } from './entity/User';
import { startJobs } from './job/schedule';
import notificationManager from './lib/notifications';
import DiscordAgent from './lib/notifications/agents/discord';
@ -49,6 +51,26 @@ app
// Load Settings
const settings = getSettings().load();
// Migrate library types
if (
settings.plex.libraries.length > 1 &&
!settings.plex.libraries[0].type
) {
const userRepository = getRepository(User);
const admin = await userRepository.findOne({
select: ['id', 'plexToken'],
order: { id: 'ASC' },
});
if (admin) {
const plexapi = new PlexAPI({ plexToken: admin.plexToken });
await plexapi.syncLibraries();
logger.info('Migrating libraries to include media type', {
label: 'Settings',
});
}
}
// Register Notification Agents
notificationManager.registerAgents([
new DiscordAgent(),

@ -1,6 +1,12 @@
import NodeCache from 'node-cache';
export type AvailableCacheIds = 'tmdb' | 'radarr' | 'sonarr' | 'rt' | 'github';
export type AvailableCacheIds =
| 'tmdb'
| 'radarr'
| 'sonarr'
| 'rt'
| 'github'
| 'plexguid';
const DEFAULT_TTL = 300;
const DEFAULT_CHECK_PERIOD = 120;
@ -48,6 +54,10 @@ class CacheManager {
stdTtl: 21600,
checkPeriod: 60 * 30,
}),
plexguid: new Cache('plexguid', 'Plex GUID Cache', {
stdTtl: 86400 * 7, // 1 week cache
checkPeriod: 60 * 30,
}),
};
public getCache(id: AvailableCacheIds): Cache {

@ -55,6 +55,7 @@ class BaseScanner<T> {
private updateRate;
protected progress = 0;
protected items: T[] = [];
protected totalSize?: number = 0;
protected scannerName: string;
protected enable4kMovie = false;
protected enable4kShow = false;
@ -609,6 +610,14 @@ class BaseScanner<T> {
): void {
logger[level](message, { label: this.scannerName, ...optional });
}
get protectedUpdateRate(): number {
return this.updateRate;
}
get protectedBundleSize(): number {
return this.bundleSize;
}
}
export default BaseScanner;

@ -4,6 +4,7 @@ import animeList from '../../../api/animelist';
import PlexAPI, { PlexLibraryItem, PlexMetadata } from '../../../api/plexapi';
import { TmdbTvDetails } from '../../../api/themoviedb/interfaces';
import { User } from '../../../entity/User';
import cacheManager from '../../cache';
import { getSettings, Library } from '../../settings';
import BaseScanner, {
MediaIds,
@ -38,7 +39,7 @@ class PlexScanner
private isRecentOnly = false;
public constructor(isRecentOnly = false) {
super('Plex Scan');
super('Plex Scan', { bundleSize: 50 });
this.isRecentOnly = isRecentOnly;
}
@ -46,7 +47,7 @@ class PlexScanner
return {
running: this.running,
progress: this.progress,
total: this.items.length,
total: this.totalSize ?? 0,
currentLibrary: this.currentLibrary,
libraries: this.libraries,
};
@ -82,10 +83,17 @@ class PlexScanner
this.currentLibrary = library;
this.log(
`Beginning to process recently added for library: ${library.name}`,
'info'
'info',
{ lastScan: library.lastScan }
);
const libraryItems = await this.plexClient.getRecentlyAdded(
library.id
library.id,
library.lastScan
? {
// We remove 10 minutes from the last scan as a buffer
addedAt: library.lastScan - 1000 * 60 * 10,
}
: undefined
);
// Bundle items up by rating keys
@ -104,13 +112,26 @@ class PlexScanner
});
await this.loop(this.processItem.bind(this), { sessionId });
// After run completes, update last scan time
const newLibraries = settings.plex.libraries.map((lib) => {
if (lib.id === library.id) {
return {
...lib,
lastScan: Date.now(),
};
}
return lib;
});
settings.plex.libraries = newLibraries;
settings.save();
}
} else {
for (const library of this.libraries) {
this.currentLibrary = library;
this.log(`Beginning to process library: ${library.name}`, 'info');
this.items = await this.plexClient.getLibraryContents(library.id);
await this.loop(this.processItem.bind(this), { sessionId });
await this.paginateLibrary(library, { sessionId });
}
}
this.log(
@ -126,6 +147,52 @@ class PlexScanner
}
}
private async paginateLibrary(
library: Library,
{ start = 0, sessionId }: { start?: number; sessionId: string }
) {
if (!this.running) {
throw new Error('Sync was aborted.');
}
if (this.sessionId !== sessionId) {
throw new Error('New session was started. Old session aborted.');
}
const response = await this.plexClient.getLibraryContents(library.id, {
size: this.protectedBundleSize,
offset: start,
});
this.progress = start;
this.totalSize = response.totalSize;
if (response.items.length === 0) {
return;
}
await Promise.all(
response.items.map(async (item) => {
await this.processItem(item);
})
);
if (response.items.length < this.protectedBundleSize) {
return;
}
await new Promise<void>((resolve, reject) =>
setTimeout(() => {
this.paginateLibrary(library, {
start: start + this.protectedBundleSize,
sessionId,
})
.then(() => resolve())
.catch((e) => reject(new Error(e.message)));
}, this.protectedUpdateRate)
);
}
private async processItem(plexitem: PlexLibraryItem) {
try {
if (plexitem.type === 'movie') {
@ -147,9 +214,8 @@ class PlexScanner
private async processPlexMovie(plexitem: PlexLibraryItem) {
const mediaIds = await this.getMediaIds(plexitem);
const metadata = await this.plexClient.getMetadata(plexitem.ratingKey);
const has4k = metadata.Media.some(
const has4k = plexitem.Media.some(
(media) => media.videoResolution === '4k'
);
@ -263,10 +329,25 @@ class PlexScanner
}
private async getMediaIds(plexitem: PlexLibraryItem): Promise<MediaIds> {
const mediaIds: Partial<MediaIds> = {};
let mediaIds: Partial<MediaIds> = {};
// Check if item is using new plex movie/tv agent
if (plexitem.guid.match(plexRegex)) {
const metadata = await this.plexClient.getMetadata(plexitem.ratingKey);
const guidCache = cacheManager.getCache('plexguid');
const cachedGuids = guidCache.data.get<MediaIds>(plexitem.ratingKey);
if (cachedGuids) {
this.log('GUIDs are cached. Skipping metadata request.', 'debug', {
mediaIds: cachedGuids,
title: plexitem.title,
});
mediaIds = cachedGuids;
}
const metadata =
plexitem.Guid && plexitem.Guid.length > 0
? plexitem
: await this.plexClient.getMetadata(plexitem.ratingKey);
// If there is no Guid field at all, then we bail
if (!metadata.Guid) {
@ -295,6 +376,10 @@ class PlexScanner
});
mediaIds.tmdbId = tmdbMovie.id;
}
// Cache GUIDs
guidCache.data.set(plexitem.ratingKey, mediaIds);
// Check if the agent is IMDb
} else if (plexitem.guid.match(imdbRegex)) {
const imdbMatch = plexitem.guid.match(imdbRegex);

@ -9,6 +9,8 @@ export interface Library {
id: string;
name: string;
enabled: boolean;
type: 'show' | 'movie';
lastScan?: number;
}
export interface Region {

@ -20,7 +20,7 @@ import { scheduledJobs } from '../../job/schedule';
import cacheManager, { AvailableCacheIds } from '../../lib/cache';
import { Permission } from '../../lib/permissions';
import { plexFullScanner } from '../../lib/scanners/plex';
import { getSettings, Library, MainSettings } from '../../lib/settings';
import { getSettings, MainSettings } from '../../lib/settings';
import logger from '../../logger';
import { isAuthenticated } from '../../middleware/auth';
import { getAppVersion } from '../../utils/appVersion';
@ -197,26 +197,7 @@ settingsRoutes.get('/plex/library', async (req, res) => {
});
const plexapi = new PlexAPI({ plexToken: admin.plexToken });
const libraries = await plexapi.getLibraries();
const newLibraries: Library[] = libraries
// Remove libraries that are not movie or show
.filter((library) => library.type === 'movie' || library.type === 'show')
// Remove libraries that do not have a metadata agent set (usually personal video libraries)
.filter((library) => library.agent !== 'com.plexapp.agents.none')
.map((library) => {
const existing = settings.plex.libraries.find(
(l) => l.id === library.key && l.name === library.title
);
return {
id: library.key,
name: library.title,
enabled: existing?.enabled ?? false,
};
});
settings.plex.libraries = newLibraries;
await plexapi.syncLibraries();
}
const enabledLibraries = req.query.enable

@ -21,6 +21,13 @@ declare module 'plex-api' {
requestOptions?: Record<string, string | number>;
});
// eslint-disable-next-line @typescript-eslint/no-explicit-any
query: <T extends Record<string, any>>(endpoint: string) => Promise<T>;
query: <T extends Record<string, any>>(
endpoint:
| string
| {
uri: string;
extraHeaders?: Record<string, string | number>;
}
) => Promise<T>;
}
}

@ -15,7 +15,8 @@
"jsx": "preserve",
"strictPropertyInitialization": false,
"experimentalDecorators": true,
"emitDecoratorMetadata": true
"emitDecoratorMetadata": true,
"useUnknownInCatchVariables": false
},
"include": ["next-env.d.ts", "src/**/*.ts", "src/**/*.tsx"],
"exclude": ["node_modules"]

@ -1247,6 +1247,18 @@
dependencies:
chalk "^4.0.0"
"@cspotcode/source-map-consumer@0.8.0":
version "0.8.0"
resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b"
integrity sha512-41qniHzTU8yAGbCp04ohlmSrZf8bkf/iJsl3V0dRGsQN/5GFfx+LbCSsCpp2gqrqjTVg/K6O8ycoV35JIwAzAg==
"@cspotcode/source-map-support@0.6.1":
version "0.6.1"
resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.6.1.tgz#118511f316e2e87ee4294761868e254d3da47960"
integrity sha512-DX3Z+T5dt1ockmPdobJS/FAsQPW4V4SrWEhD2iYQT2Cb2tQsiMnYxrcUH9By/Z3B+v0S5LMBkQtV/XOBbpLEOg==
dependencies:
"@cspotcode/source-map-consumer" "0.8.0"
"@dabh/diagnostics@^2.0.2":
version "2.0.2"
resolved "https://registry.yarnpkg.com/@dabh/diagnostics/-/diagnostics-2.0.2.tgz#290d08f7b381b8f94607dc8f471a12c675f9db31"
@ -2164,10 +2176,10 @@
resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.1.tgz#95f2d167ffb9b8d2068b0b235302fafd4df711f2"
integrity sha512-509r2+yARFfHHE7T6Puu2jjkoycftovhXRqW328PDXTVGKihlb1P8Z9mMZH04ebyajfRY7dedfGynlrFHJUQCg==
"@tsconfig/node16@^1.0.1":
version "1.0.1"
resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.1.tgz#a6ca6a9a0ff366af433f42f5f0e124794ff6b8f1"
integrity sha512-FTgBI767POY/lKNDNbIzgAX6miIDBs6NTCbdlDb8TrWovHsSvaVIZDlTqym29C6UqhzwcJx4CYr+AlrMywA0cA==
"@tsconfig/node16@^1.0.2":
version "1.0.2"
resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.2.tgz#423c77877d0569db20e1fc80885ac4118314010e"
integrity sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA==
"@types/babel__core@^7.1.7":
version "7.1.9"
@ -2808,6 +2820,11 @@ acorn-walk@^7.0.0:
resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc"
integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==
acorn-walk@^8.1.1:
version "8.2.0"
resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1"
integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==
acorn@^7.0.0, acorn@^7.1.1:
version "7.4.1"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa"
@ -2818,6 +2835,11 @@ acorn@^7.4.0:
resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.0.tgz#e1ad486e6c54501634c6c397c5c121daa383607c"
integrity sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==
acorn@^8.4.1:
version "8.5.0"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.5.0.tgz#4512ccb99b3698c752591e9bb4472e38ad43cee2"
integrity sha512-yXbYeFy+jUuYd3/CDcg2NkIYE991XYX/bje7LmjJigUciaeO1JR4XxXgCIV1/Zc/dRuFEyw1L0pbA+qynJkW5Q==
agent-base@4, agent-base@^4.3.0:
version "4.3.0"
resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.3.0.tgz#8165f01c436009bccad0b1d122f05ed770efc6ee"
@ -12695,14 +12717,6 @@ source-map-resolve@^0.5.0:
source-map-url "^0.4.0"
urix "^0.1.0"
source-map-support@^0.5.17:
version "0.5.19"
resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.19.tgz#a98b62f86dcaf4f67399648c085291ab9e8fed61"
integrity sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==
dependencies:
buffer-from "^1.0.0"
source-map "^0.6.0"
source-map-url@^0.4.0:
version "0.4.0"
resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.0.tgz#3e935d7ddd73631b97659956d55128e87b5084a3"
@ -12730,7 +12744,7 @@ source-map@^0.5.0, source-map@^0.5.6, source-map@^0.5.7:
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc"
integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=
source-map@^0.6.0, source-map@^0.6.1:
source-map@^0.6.1:
version "0.6.1"
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
@ -13651,20 +13665,22 @@ ts-easing@^0.2.0:
resolved "https://registry.yarnpkg.com/ts-easing/-/ts-easing-0.2.0.tgz#c8a8a35025105566588d87dbda05dd7fbfa5a4ec"
integrity sha512-Z86EW+fFFh/IFB1fqQ3/+7Zpf9t2ebOAxNI/V6Wo7r5gqiqtxmgTlQ1qbqQcjLKYeSHPTsEmvlJUDg/EuL0uHQ==
ts-node@^10.0.0:
version "10.0.0"
resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.0.0.tgz#05f10b9a716b0b624129ad44f0ea05dac84ba3be"
integrity sha512-ROWeOIUvfFbPZkoDis0L/55Fk+6gFQNZwwKPLinacRl6tsxstTF1DbAcLKkovwnpKMVvOMHP1TIbnwXwtLg1gg==
ts-node@^10.2.1:
version "10.2.1"
resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.2.1.tgz#4cc93bea0a7aba2179497e65bb08ddfc198b3ab5"
integrity sha512-hCnyOyuGmD5wHleOQX6NIjJtYVIO8bPP8F2acWkB4W06wdlkgyvJtubO/I9NkI88hCFECbsEgoLc0VNkYmcSfw==
dependencies:
"@cspotcode/source-map-support" "0.6.1"
"@tsconfig/node10" "^1.0.7"
"@tsconfig/node12" "^1.0.7"
"@tsconfig/node14" "^1.0.0"
"@tsconfig/node16" "^1.0.1"
"@tsconfig/node16" "^1.0.2"
acorn "^8.4.1"
acorn-walk "^8.1.1"
arg "^4.1.0"
create-require "^1.1.0"
diff "^4.0.1"
make-error "^1.1.1"
source-map-support "^0.5.17"
yn "3.1.1"
ts-pnp@^1.1.6:
@ -13828,10 +13844,10 @@ typescript@^4.0:
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.0.2.tgz#7ea7c88777c723c681e33bf7988be5d008d05ac2"
integrity sha512-e4ERvRV2wb+rRZ/IQeb3jm2VxBsirQLpQhdxplZ2MEzGvDkkMmPglecnNDfSUBivMjP93vRbngYYDQqQ/78bcQ==
typescript@^4.3.4:
version "4.3.4"
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.3.4.tgz#3f85b986945bcf31071decdd96cf8bfa65f9dcbc"
integrity sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew==
typescript@^4.4.3:
version "4.4.3"
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.4.3.tgz#bdc5407caa2b109efd4f82fe130656f977a29324"
integrity sha512-4xfscpisVgqqDfPaJo5vkd+Qd/ItkoagnHpufr+i2QCHBsNYp+G7UAoyFl8aPtx879u38wPV65rZ8qbGZijalA==
uc.micro@^1.0.1:
version "1.0.6"

Loading…
Cancel
Save