From 1390cc1f130bb3975996e84b12ac833f55f2f753 Mon Sep 17 00:00:00 2001 From: sct Date: Wed, 11 Nov 2020 09:02:28 +0000 Subject: [PATCH] feat(api): plex tv sync and recently added sync --- .gitignore | 3 + overseerr-api.yml | 23 +++ server/api/plexapi.ts | 35 +++- server/api/themoviedb.ts | 32 ++++ server/entity/Media.ts | 7 + server/entity/Season.ts | 37 ++++ server/job/{plexsync.ts => plexsync/index.ts} | 158 +++++++++++++++--- server/job/schedule.ts | 28 +++- server/routes/settings.ts | 18 +- server/types/plex-api.d.ts | 1 + src/components/Layout/UserDropdown/index.tsx | 14 -- .../RequestModal/TvRequestModal.tsx | 68 +++++--- src/components/Settings/LibraryItem.tsx | 4 - src/components/Settings/SettingsJobs.tsx | 70 ++++++++ src/components/UserList/index.tsx | 10 +- src/components/UserProfile/index.tsx | 96 +++++++++++ src/hooks/useUser.ts | 3 +- src/pages/settings/jobs.tsx | 14 ++ src/pages/users/[userId].tsx | 9 + 19 files changed, 554 insertions(+), 76 deletions(-) create mode 100644 server/entity/Season.ts rename server/job/{plexsync.ts => plexsync/index.ts} (50%) create mode 100644 src/components/Settings/SettingsJobs.tsx create mode 100644 src/components/UserProfile/index.tsx create mode 100644 src/pages/settings/jobs.tsx create mode 100644 src/pages/users/[userId].tsx diff --git a/.gitignore b/.gitignore index 9d3e6fc48..ebd30895f 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,6 @@ config/logs/*.log # dist files dist + +# sqlite journal +config/db/db.sqlite3-journal diff --git a/overseerr-api.yml b/overseerr-api.yml index be0a431ce..5647bb5ed 100644 --- a/overseerr-api.yml +++ b/overseerr-api.yml @@ -1155,6 +1155,29 @@ paths: application/json: schema: $ref: '#/components/schemas/PublicSettings' + /settings/jobs: + get: + summary: Returns list of scheduled jobs + description: Returns list of all scheduled jobs and details about their next execution time + tags: + - settings + responses: + '200': + description: Scheduled jobs returned + content: + application/json: + schema: + type: array + items: + type: object + properties: + name: + type: string + example: A Job Name + nextExecutionTime: + type: string + example: '2020-09-02T05:02:23.000Z' + /auth/me: get: summary: Returns the currently logged in user diff --git a/server/api/plexapi.ts b/server/api/plexapi.ts index 7bc1175f7..d38a77cd6 100644 --- a/server/api/plexapi.ts +++ b/server/api/plexapi.ts @@ -3,9 +3,11 @@ import { getSettings } from '../lib/settings'; export interface PlexLibraryItem { ratingKey: string; + parentRatingKey?: string; title: string; guid: string; - type: 'movie' | 'show'; + parentGuid?: string; + type: 'movie' | 'show' | 'season'; } interface PlexLibraryResponse { @@ -28,12 +30,21 @@ interface PlexLibrariesResponse { export interface PlexMetadata { ratingKey: string; + parentRatingKey?: string; guid: string; - type: 'movie' | 'show'; + type: 'movie' | 'show' | 'season'; title: string; Guid: { id: string; }[]; + Children?: { + size: 12; + Metadata: PlexMetadata[]; + }; + index: number; + parentIndex?: number; + leafCount: number; + viewedLeafCount: number; } interface PlexMetadataResponse { @@ -63,6 +74,9 @@ class PlexAPI { cb(undefined, plexToken); }, }, + // requestOptions: { + // includeChildren: 1, + // }, options: { identifier: settings.clientId, product: 'Overseerr', @@ -92,18 +106,25 @@ class PlexAPI { return response.MediaContainer.Metadata; } - public async getMetadata(key: string): Promise { + public async getMetadata( + key: string, + options: { includeChildren?: boolean } = {} + ): Promise { const response = await this.plexClient.query( - `/library/metadata/${key}` + `/library/metadata/${key}${ + options.includeChildren ? '?includeChildren=1' : '' + }` ); return response.MediaContainer.Metadata[0]; } - public async getRecentlyAdded() { - const response = await this.plexClient.query('/library/recentlyAdded'); + public async getRecentlyAdded(): Promise { + const response = await this.plexClient.query( + '/library/recentlyAdded' + ); - return response; + return response.MediaContainer.Metadata; } } diff --git a/server/api/themoviedb.ts b/server/api/themoviedb.ts index 337db3714..05d5aee84 100644 --- a/server/api/themoviedb.ts +++ b/server/api/themoviedb.ts @@ -649,6 +649,38 @@ class TheMovieDb { ); } } + + public async getShowByTvdbId({ + tvdbId, + language = 'en-US', + }: { + tvdbId: number; + language?: string; + }): Promise { + try { + const extResponse = await this.getByExternalId({ + externalId: tvdbId, + type: 'tvdb', + }); + + if (extResponse.tv_results[0]) { + const tvshow = await this.getTvShow({ + tvId: extResponse.tv_results[0].id, + language, + }); + + return tvshow; + } + + throw new Error( + `[TMDB] Failed to find a tv show with the provided TVDB id: ${tvdbId}` + ); + } catch (e) { + throw new Error( + `[TMDB] Failed to get tv show by external tvdb ID: ${e.message}` + ); + } + } } export default TheMovieDb; diff --git a/server/entity/Media.ts b/server/entity/Media.ts index 0cb616a11..0222e1043 100644 --- a/server/entity/Media.ts +++ b/server/entity/Media.ts @@ -12,6 +12,7 @@ import { import { MediaRequest } from './MediaRequest'; import { MediaStatus, MediaType } from '../constants/media'; import logger from '../logger'; +import Season from './Season'; @Entity() class Media { @@ -79,6 +80,12 @@ class Media { @OneToMany(() => MediaRequest, (request) => request.media, { cascade: true }) public requests: MediaRequest[]; + @OneToMany(() => Season, (season) => season.media, { + cascade: true, + eager: true, + }) + public seasons: Season[]; + @CreateDateColumn() public createdAt: Date; diff --git a/server/entity/Season.ts b/server/entity/Season.ts new file mode 100644 index 000000000..3f18ed08f --- /dev/null +++ b/server/entity/Season.ts @@ -0,0 +1,37 @@ +import { + Entity, + PrimaryGeneratedColumn, + Column, + ManyToOne, + CreateDateColumn, + UpdateDateColumn, +} from 'typeorm'; +import { MediaStatus } from '../constants/media'; +import Media from './Media'; + +@Entity() +class Season { + @PrimaryGeneratedColumn() + public id: number; + + @Column() + public seasonNumber: number; + + @Column({ type: 'int', default: MediaStatus.UNKNOWN }) + public status: MediaStatus; + + @ManyToOne(() => Media, (media) => media.seasons) + public media: Media; + + @CreateDateColumn() + public createdAt: Date; + + @UpdateDateColumn() + public updatedAt: Date; + + constructor(init?: Partial) { + Object.assign(this, init); + } +} + +export default Season; diff --git a/server/job/plexsync.ts b/server/job/plexsync/index.ts similarity index 50% rename from server/job/plexsync.ts rename to server/job/plexsync/index.ts index 90e282ef0..ee70f3b90 100644 --- a/server/job/plexsync.ts +++ b/server/job/plexsync/index.ts @@ -1,16 +1,19 @@ import { getRepository } from 'typeorm'; -import { User } from '../entity/User'; -import PlexAPI, { PlexLibraryItem } from '../api/plexapi'; -import TheMovieDb from '../api/themoviedb'; -import Media from '../entity/Media'; -import { MediaStatus, MediaType } from '../constants/media'; -import logger from '../logger'; -import { getSettings, Library } from '../lib/settings'; +import { User } from '../../entity/User'; +import PlexAPI, { PlexLibraryItem } from '../../api/plexapi'; +import TheMovieDb, { TmdbTvDetails } from '../../api/themoviedb'; +import Media from '../../entity/Media'; +import { MediaStatus, MediaType } from '../../constants/media'; +import logger from '../../logger'; +import { getSettings, Library } from '../../lib/settings'; +import Season from '../../entity/Season'; const BUNDLE_SIZE = 10; const imdbRegex = new RegExp(/imdb:\/\/(tt[0-9]+)/); const tmdbRegex = new RegExp(/tmdb:\/\/([0-9]+)/); +const tvdbRegex = new RegExp(/tvdb:\/\/([0-9]+)/); +const tmdbShowRegex = new RegExp(/themoviedb:\/\/([0-9]+)/); const plexRegex = new RegExp(/plex:\/\//); interface SyncStatus { @@ -29,9 +32,11 @@ class JobPlexSync { private libraries: Library[]; private currentLibrary: Library; private running = false; + private isRecentOnly = false; - constructor() { + constructor({ isRecentOnly }: { isRecentOnly?: boolean } = {}) { this.tmdb = new TheMovieDb(); + this.isRecentOnly = isRecentOnly ?? false; } private async getExisting(tmdbId: number) { @@ -107,11 +112,116 @@ class JobPlexSync { } } + private async processShow(plexitem: PlexLibraryItem) { + const mediaRepository = getRepository(Media); + + let tvShow: TmdbTvDetails | null = null; + + try { + const metadata = await this.plexClient.getMetadata( + plexitem.parentRatingKey ?? plexitem.ratingKey, + { includeChildren: true } + ); + if (metadata.guid.match(tvdbRegex)) { + const matchedtvdb = metadata.guid.match(tvdbRegex); + + // If we can find a tvdb Id, use it to get the full tmdb show details + if (matchedtvdb?.[1]) { + tvShow = await this.tmdb.getShowByTvdbId({ + tvdbId: Number(matchedtvdb[1]), + }); + } + } else if (metadata.guid.match(tmdbShowRegex)) { + const matchedtmdb = metadata.guid.match(tmdbShowRegex); + + if (matchedtmdb?.[1]) { + tvShow = await this.tmdb.getTvShow({ tvId: Number(matchedtmdb[1]) }); + } + } + + if (tvShow && metadata) { + // Lets get the available seasons from plex + const seasons = tvShow.seasons; + const media = await mediaRepository.findOne({ + where: { tmdbId: tvShow.id, mediaType: MediaType.TV }, + }); + + const availableSeasons: Season[] = []; + + seasons.forEach((season) => { + const matchedPlexSeason = metadata.Children?.Metadata.find( + (md) => Number(md.index) === season.season_number + ); + + // Check if we found the matching season and it has all the available episodes + if ( + matchedPlexSeason && + Number(matchedPlexSeason.leafCount) === season.episode_count + ) { + availableSeasons.push( + new Season({ + seasonNumber: season.season_number, + status: MediaStatus.AVAILABLE, + }) + ); + } else if (matchedPlexSeason) { + availableSeasons.push( + new Season({ + seasonNumber: season.season_number, + status: MediaStatus.PARTIALLY_AVAILABLE, + }) + ); + } + }); + + // Remove extras season. We dont count it for determining availability + const filteredSeasons = tvShow.seasons.filter( + (season) => season.season_number !== 0 + ); + + const isAllSeasons = availableSeasons.length >= filteredSeasons.length; + + if (media) { + // Update existing + media.seasons = availableSeasons; + media.status = isAllSeasons + ? MediaStatus.AVAILABLE + : MediaStatus.PARTIALLY_AVAILABLE; + await mediaRepository.save(media); + this.log(`Updating existing title: ${tvShow.name}`); + } else { + const newMedia = new Media({ + mediaType: MediaType.TV, + seasons: availableSeasons, + tmdbId: tvShow.id, + tvdbId: tvShow.external_ids.tvdb_id, + status: isAllSeasons + ? MediaStatus.AVAILABLE + : MediaStatus.PARTIALLY_AVAILABLE, + }); + await mediaRepository.save(newMedia); + this.log(`Saved ${tvShow.name}`); + } + } else { + this.log(`failed show: ${plexitem.guid}`); + } + } catch (e) { + this.log( + `Failed to process plex item. ratingKey: ${ + plexitem.parentRatingKey ?? plexitem.ratingKey + }`, + 'error' + ); + } + } + private async processItems(slicedItems: PlexLibraryItem[]) { await Promise.all( slicedItems.map(async (plexitem) => { if (plexitem.type === 'movie') { await this.processMovie(plexitem); + } else if (plexitem.type === 'show') { + await this.processShow(plexitem); } }) ); @@ -159,15 +269,26 @@ class JobPlexSync { }); this.plexClient = new PlexAPI({ plexToken: admin.plexToken }); - this.libraries = settings.plex.libraries.filter( - (library) => library.enabled - ); - - for (const library of this.libraries) { - this.currentLibrary = library; - this.log(`Beginning to process library: ${library.name}`, 'info'); - this.items = await this.plexClient.getLibraryContents(library.id); + if (this.isRecentOnly) { + this.currentLibrary = { + id: '0', + name: 'Recently Added', + enabled: true, + }; + this.log(`Beginning to process recently added`, 'info'); + this.items = await this.plexClient.getRecentlyAdded(); await this.loop(); + } else { + this.libraries = settings.plex.libraries.filter( + (library) => library.enabled + ); + + for (const library of this.libraries) { + this.currentLibrary = library; + this.log(`Beginning to process library: ${library.name}`, 'info'); + this.items = await this.plexClient.getLibraryContents(library.id); + await this.loop(); + } } this.running = false; this.log('complete'); @@ -189,6 +310,5 @@ class JobPlexSync { } } -const jobPlexSync = new JobPlexSync(); - -export default jobPlexSync; +export const jobPlexFullSync = new JobPlexSync(); +export const jobPlexRecentSync = new JobPlexSync({ isRecentOnly: true }); diff --git a/server/job/schedule.ts b/server/job/schedule.ts index 5589a109c..8dce703b4 100644 --- a/server/job/schedule.ts +++ b/server/job/schedule.ts @@ -1,14 +1,32 @@ import schedule from 'node-schedule'; -import jobPlexSync from './plexsync'; +import { jobPlexFullSync, jobPlexRecentSync } from './plexsync'; import logger from '../logger'; -export const scheduledJobs: Record = {}; +interface ScheduledJob { + job: schedule.Job; + name: string; +} + +export const scheduledJobs: ScheduledJob[] = []; export const startJobs = (): void => { + // Run recently added plex sync every 5 minutes + scheduledJobs.push({ + name: 'Plex Recently Added Sync', + job: schedule.scheduleJob('0 */10 * * * *', () => { + logger.info('Starting scheduled job: Plex Recently Added Sync', { + label: 'Jobs', + }); + jobPlexRecentSync.run(); + }), + }); // Run full plex sync every 6 hours - scheduledJobs.plexFullSync = schedule.scheduleJob('* */6 * * *', () => { - logger.info('Starting scheduled job: Plex Full Sync', { label: 'Jobs' }); - jobPlexSync.run(); + scheduledJobs.push({ + name: 'Plex Full Library Sync', + job: schedule.scheduleJob('* * */6 * * *', () => { + logger.info('Starting scheduled job: Plex Full Sync', { label: 'Jobs' }); + jobPlexFullSync.run(); + }), }); logger.info('Scheduled jobs loaded', { label: 'Jobs' }); diff --git a/server/routes/settings.ts b/server/routes/settings.ts index 523e29f7e..e1b992ede 100644 --- a/server/routes/settings.ts +++ b/server/routes/settings.ts @@ -8,10 +8,11 @@ import { import { getRepository } from 'typeorm'; import { User } from '../entity/User'; import PlexAPI, { PlexLibrary } from '../api/plexapi'; -import jobPlexSync from '../job/plexsync'; +import { jobPlexFullSync } from '../job/plexsync'; import SonarrAPI from '../api/sonarr'; import RadarrAPI from '../api/radarr'; import logger from '../logger'; +import { scheduledJobs } from '../job/schedule'; const settingsRoutes = Router(); @@ -108,12 +109,12 @@ settingsRoutes.get('/plex/library', async (req, res) => { settingsRoutes.get('/plex/sync', (req, res) => { if (req.query.cancel) { - jobPlexSync.cancel(); + jobPlexFullSync.cancel(); } else if (req.query.start) { - jobPlexSync.run(); + jobPlexFullSync.run(); } - return res.status(200).json(jobPlexSync.status()); + return res.status(200).json(jobPlexFullSync.status()); }); settingsRoutes.get('/radarr', (req, res) => { @@ -324,4 +325,13 @@ settingsRoutes.delete<{ id: string }>('/sonarr/:id', (req, res) => { return res.status(200).json(removed[0]); }); +settingsRoutes.get('/jobs', (req, res) => { + return res.status(200).json( + scheduledJobs.map((job) => ({ + name: job.name, + nextExecutionTime: job.job.nextInvocation(), + })) + ); +}); + export default settingsRoutes; diff --git a/server/types/plex-api.d.ts b/server/types/plex-api.d.ts index 8a9621832..2c4801676 100644 --- a/server/types/plex-api.d.ts +++ b/server/types/plex-api.d.ts @@ -16,6 +16,7 @@ declare module 'plex-api' { deviceName: string; platform: string; }; + requestOptions?: Record; }); query: >(endpoint: string) => Promise; diff --git a/src/components/Layout/UserDropdown/index.tsx b/src/components/Layout/UserDropdown/index.tsx index 8f4504749..2fe318631 100644 --- a/src/components/Layout/UserDropdown/index.tsx +++ b/src/components/Layout/UserDropdown/index.tsx @@ -50,20 +50,6 @@ const UserDropdown: React.FC = () => { aria-orientation="vertical" aria-labelledby="user-menu" > - - Your Profile - - - Settings - = ({ } }; - const getAllRequestedSeasons = (): number[] => - (data?.mediaInfo?.requests ?? []).reduce((requestedSeasons, request) => { - return [ - ...requestedSeasons, - ...request.seasons.map((sr) => sr.seasonNumber), - ]; - }, [] as number[]); + const getAllRequestedSeasons = (): number[] => { + const requestedSeasons = (data?.mediaInfo?.requests ?? []).reduce( + (requestedSeasons, request) => { + return [ + ...requestedSeasons, + ...request.seasons.map((sr) => sr.seasonNumber), + ]; + }, + [] as number[] + ); + + const availableSeasons = (data?.mediaInfo?.seasons ?? []) + .filter( + (season) => + (season.status === MediaStatus.AVAILABLE || + season.status === MediaStatus.PARTIALLY_AVAILABLE) && + !requestedSeasons.includes(season.seasonNumber) + ) + .map((season) => season.seasonNumber); - const isSelectedSeason = (seasonNumber: number): boolean => { - return selectedSeasons.includes(seasonNumber); + return [...requestedSeasons, ...availableSeasons]; }; + const isSelectedSeason = (seasonNumber: number): boolean => + selectedSeasons.includes(seasonNumber); + const toggleSeason = (seasonNumber: number): void => { // If this season already has a pending request, don't allow it to be toggled if (getAllRequestedSeasons().includes(seasonNumber)) { @@ -241,6 +255,9 @@ const TvRequestModal: React.FC = ({ const seasonRequest = getSeasonRequest( season.seasonNumber ); + const mediaSeason = data?.mediaInfo?.seasons.find( + (sn) => sn.seasonNumber === season.seasonNumber + ); return ( @@ -248,6 +265,7 @@ const TvRequestModal: React.FC = ({ role="checkbox" tabIndex={0} aria-checked={ + !!mediaSeason || !!seasonRequest || isSelectedSeason(season.seasonNumber) } @@ -258,12 +276,13 @@ const TvRequestModal: React.FC = ({ } }} className={`group relative inline-flex items-center justify-center flex-shrink-0 h-5 w-10 cursor-pointer focus:outline-none ${ - seasonRequest ? 'opacity-50' : '' + mediaSeason || seasonRequest ? 'opacity-50' : '' }`} >