From a51d2a24d51d092a0c6da608e3322f19a37c2d28 Mon Sep 17 00:00:00 2001 From: sct Date: Thu, 11 Mar 2021 12:19:11 +0900 Subject: [PATCH] feat(scan): add support for new plex tv agent (#1144) --- server/job/plexsync/index.ts | 926 ---------------------------- server/job/radarrsync/index.ts | 248 -------- server/job/schedule.ts | 30 +- server/job/sonarrsync/index.ts | 381 ------------ server/lib/scanners/baseScanner.ts | 616 ++++++++++++++++++ server/lib/scanners/plex/index.ts | 463 ++++++++++++++ server/lib/scanners/radarr/index.ts | 94 +++ server/lib/scanners/sonarr/index.ts | 134 ++++ server/routes/settings/index.ts | 10 +- 9 files changed, 1327 insertions(+), 1575 deletions(-) delete mode 100644 server/job/plexsync/index.ts delete mode 100644 server/job/radarrsync/index.ts delete mode 100644 server/job/sonarrsync/index.ts create mode 100644 server/lib/scanners/baseScanner.ts create mode 100644 server/lib/scanners/plex/index.ts create mode 100644 server/lib/scanners/radarr/index.ts create mode 100644 server/lib/scanners/sonarr/index.ts diff --git a/server/job/plexsync/index.ts b/server/job/plexsync/index.ts deleted file mode 100644 index 60840e0b..00000000 --- a/server/job/plexsync/index.ts +++ /dev/null @@ -1,926 +0,0 @@ -import { getRepository } from 'typeorm'; -import { User } from '../../entity/User'; -import PlexAPI, { PlexLibraryItem, PlexMetadata } from '../../api/plexapi'; -import TheMovieDb from '../../api/themoviedb'; -import { - TmdbMovieDetails, - TmdbTvDetails, -} from '../../api/themoviedb/interfaces'; -import Media from '../../entity/Media'; -import { MediaStatus, MediaType } from '../../constants/media'; -import logger from '../../logger'; -import { getSettings, Library } from '../../lib/settings'; -import Season from '../../entity/Season'; -import { uniqWith } from 'lodash'; -import { v4 as uuid } from 'uuid'; -import animeList from '../../api/animelist'; -import AsyncLock from '../../utils/asyncLock'; - -const BUNDLE_SIZE = 20; -const UPDATE_RATE = 4 * 1000; - -const imdbRegex = new RegExp(/imdb:\/\/(tt[0-9]+)/); -const tmdbRegex = new RegExp(/tmdb:\/\/([0-9]+)/); -const tvdbRegex = new RegExp(/tvdb:\/\/([0-9]+)/); -const tmdbShowRegex = new RegExp(/themoviedb:\/\/([0-9]+)/); -const plexRegex = new RegExp(/plex:\/\//); -// Hama agent uses ASS naming, see details here: -// https://github.com/ZeroQI/Absolute-Series-Scanner/blob/master/README.md#forcing-the-movieseries-id -const hamaTvdbRegex = new RegExp(/hama:\/\/tvdb[0-9]?-([0-9]+)/); -const hamaAnidbRegex = new RegExp(/hama:\/\/anidb[0-9]?-([0-9]+)/); -const HAMA_AGENT = 'com.plexapp.agents.hama'; - -interface SyncStatus { - running: boolean; - progress: number; - total: number; - currentLibrary: Library; - libraries: Library[]; -} - -class JobPlexSync { - private sessionId: string; - private tmdb: TheMovieDb; - private plexClient: PlexAPI; - private items: PlexLibraryItem[] = []; - private progress = 0; - private libraries: Library[]; - private currentLibrary: Library; - private running = false; - private isRecentOnly = false; - private enable4kMovie = false; - private enable4kShow = false; - private asyncLock = new AsyncLock(); - - constructor({ isRecentOnly }: { isRecentOnly?: boolean } = {}) { - this.tmdb = new TheMovieDb(); - this.isRecentOnly = isRecentOnly ?? false; - } - - private async getExisting(tmdbId: number, mediaType: MediaType) { - const mediaRepository = getRepository(Media); - - const existing = await mediaRepository.findOne({ - where: { tmdbId: tmdbId, mediaType }, - }); - - return existing; - } - - private async processMovie(plexitem: PlexLibraryItem) { - const mediaRepository = getRepository(Media); - - try { - if (plexitem.guid.match(plexRegex)) { - const metadata = await this.plexClient.getMetadata(plexitem.ratingKey); - const newMedia = new Media(); - - if (!metadata.Guid) { - logger.debug('No Guid metadata for this title. Skipping', { - label: 'Plex Scan', - ratingKey: plexitem.ratingKey, - }); - return; - } - - metadata.Guid.forEach((ref) => { - if (ref.id.match(imdbRegex)) { - newMedia.imdbId = ref.id.match(imdbRegex)?.[1] ?? undefined; - } else if (ref.id.match(tmdbRegex)) { - const tmdbMatch = ref.id.match(tmdbRegex)?.[1]; - newMedia.tmdbId = Number(tmdbMatch); - } - }); - if (newMedia.imdbId && !newMedia.tmdbId) { - const tmdbMovie = await this.tmdb.getMovieByImdbId({ - imdbId: newMedia.imdbId, - }); - newMedia.tmdbId = tmdbMovie.id; - } - if (!newMedia.tmdbId) { - throw new Error('Unable to find TMDb ID'); - } - - const has4k = metadata.Media.some( - (media) => media.videoResolution === '4k' - ); - const hasOtherResolution = metadata.Media.some( - (media) => media.videoResolution !== '4k' - ); - - await this.asyncLock.dispatch(newMedia.tmdbId, async () => { - const existing = await this.getExisting( - newMedia.tmdbId, - MediaType.MOVIE - ); - - if (existing) { - let changedExisting = false; - - if ( - (hasOtherResolution || (!this.enable4kMovie && has4k)) && - existing.status !== MediaStatus.AVAILABLE - ) { - existing.status = MediaStatus.AVAILABLE; - existing.mediaAddedAt = new Date(plexitem.addedAt * 1000); - changedExisting = true; - } - - if ( - has4k && - this.enable4kMovie && - existing.status4k !== MediaStatus.AVAILABLE - ) { - existing.status4k = MediaStatus.AVAILABLE; - changedExisting = true; - } - - if (!existing.mediaAddedAt && !changedExisting) { - existing.mediaAddedAt = new Date(plexitem.addedAt * 1000); - changedExisting = true; - } - - if ( - (hasOtherResolution || (has4k && !this.enable4kMovie)) && - existing.ratingKey !== plexitem.ratingKey - ) { - existing.ratingKey = plexitem.ratingKey; - changedExisting = true; - } - - if ( - has4k && - this.enable4kMovie && - existing.ratingKey4k !== plexitem.ratingKey - ) { - existing.ratingKey4k = plexitem.ratingKey; - changedExisting = true; - } - - if (changedExisting) { - await mediaRepository.save(existing); - this.log( - `Request for ${metadata.title} exists. New media types set to AVAILABLE`, - 'info' - ); - } else { - this.log( - `Title already exists and no new media types found ${metadata.title}` - ); - } - } else { - newMedia.status = - hasOtherResolution || (!this.enable4kMovie && has4k) - ? MediaStatus.AVAILABLE - : MediaStatus.UNKNOWN; - newMedia.status4k = - has4k && this.enable4kMovie - ? MediaStatus.AVAILABLE - : MediaStatus.UNKNOWN; - newMedia.mediaType = MediaType.MOVIE; - newMedia.mediaAddedAt = new Date(plexitem.addedAt * 1000); - newMedia.ratingKey = - hasOtherResolution || (!this.enable4kMovie && has4k) - ? plexitem.ratingKey - : undefined; - newMedia.ratingKey4k = - has4k && this.enable4kMovie ? plexitem.ratingKey : undefined; - await mediaRepository.save(newMedia); - this.log(`Saved ${plexitem.title}`); - } - }); - } else { - let tmdbMovieId: number | undefined; - let tmdbMovie: TmdbMovieDetails | undefined; - - const imdbMatch = plexitem.guid.match(imdbRegex); - const tmdbMatch = plexitem.guid.match(tmdbShowRegex); - - if (imdbMatch) { - tmdbMovie = await this.tmdb.getMovieByImdbId({ - imdbId: imdbMatch[1], - }); - tmdbMovieId = tmdbMovie.id; - } else if (tmdbMatch) { - tmdbMovieId = Number(tmdbMatch[1]); - } - - if (!tmdbMovieId) { - throw new Error('Unable to find TMDb ID'); - } - - await this.processMovieWithId(plexitem, tmdbMovie, tmdbMovieId); - } - } catch (e) { - this.log( - `Failed to process Plex item. ratingKey: ${plexitem.ratingKey}`, - 'error', - { - errorMessage: e.message, - plexitem, - } - ); - } - } - - private async processMovieWithId( - plexitem: PlexLibraryItem, - tmdbMovie: TmdbMovieDetails | undefined, - tmdbMovieId: number - ) { - const mediaRepository = getRepository(Media); - - await this.asyncLock.dispatch(tmdbMovieId, async () => { - const metadata = await this.plexClient.getMetadata(plexitem.ratingKey); - const existing = await this.getExisting(tmdbMovieId, MediaType.MOVIE); - - const has4k = metadata.Media.some( - (media) => media.videoResolution === '4k' - ); - const hasOtherResolution = metadata.Media.some( - (media) => media.videoResolution !== '4k' - ); - - if (existing) { - let changedExisting = false; - - if ( - (hasOtherResolution || (!this.enable4kMovie && has4k)) && - existing.status !== MediaStatus.AVAILABLE - ) { - existing.status = MediaStatus.AVAILABLE; - existing.mediaAddedAt = new Date(plexitem.addedAt * 1000); - changedExisting = true; - } - - if ( - has4k && - this.enable4kMovie && - existing.status4k !== MediaStatus.AVAILABLE - ) { - existing.status4k = MediaStatus.AVAILABLE; - changedExisting = true; - } - - if (!existing.mediaAddedAt && !changedExisting) { - existing.mediaAddedAt = new Date(plexitem.addedAt * 1000); - changedExisting = true; - } - - if ( - (hasOtherResolution || (has4k && !this.enable4kMovie)) && - existing.ratingKey !== plexitem.ratingKey - ) { - existing.ratingKey = plexitem.ratingKey; - changedExisting = true; - } - - if ( - has4k && - this.enable4kMovie && - existing.ratingKey4k !== plexitem.ratingKey - ) { - existing.ratingKey4k = plexitem.ratingKey; - changedExisting = true; - } - - if (changedExisting) { - await mediaRepository.save(existing); - this.log( - `Request for ${metadata.title} exists. New media types set to AVAILABLE`, - 'info' - ); - } else { - this.log( - `Title already exists and no new media types found ${metadata.title}` - ); - } - } else { - // If we have a tmdb movie guid but it didn't already exist, only then - // do we request the movie from tmdb (to reduce api requests) - if (!tmdbMovie) { - tmdbMovie = await this.tmdb.getMovie({ movieId: tmdbMovieId }); - } - const newMedia = new Media(); - newMedia.imdbId = tmdbMovie.external_ids.imdb_id; - newMedia.tmdbId = tmdbMovie.id; - newMedia.mediaAddedAt = new Date(plexitem.addedAt * 1000); - newMedia.status = - hasOtherResolution || (!this.enable4kMovie && has4k) - ? MediaStatus.AVAILABLE - : MediaStatus.UNKNOWN; - newMedia.status4k = - has4k && this.enable4kMovie - ? MediaStatus.AVAILABLE - : MediaStatus.UNKNOWN; - newMedia.mediaType = MediaType.MOVIE; - newMedia.ratingKey = - hasOtherResolution || (!this.enable4kMovie && has4k) - ? plexitem.ratingKey - : undefined; - newMedia.ratingKey4k = - has4k && this.enable4kMovie ? plexitem.ratingKey : undefined; - await mediaRepository.save(newMedia); - this.log(`Saved ${tmdbMovie.title}`); - } - }); - } - - // this adds all movie episodes from specials season for Hama agent - private async processHamaSpecials(metadata: PlexMetadata, tvdbId: number) { - const specials = metadata.Children?.Metadata.find( - (md) => Number(md.index) === 0 - ); - if (specials) { - const episodes = await this.plexClient.getChildrenMetadata( - specials.ratingKey - ); - if (episodes) { - for (const episode of episodes) { - const special = animeList.getSpecialEpisode(tvdbId, episode.index); - if (special) { - if (special.tmdbId) { - await this.processMovieWithId(episode, undefined, special.tmdbId); - } else if (special.imdbId) { - const tmdbMovie = await this.tmdb.getMovieByImdbId({ - imdbId: special.imdbId, - }); - await this.processMovieWithId(episode, tmdbMovie, tmdbMovie.id); - } - } - } - } - } - } - - // movies with hama agent actually are tv shows with at least one episode in it - // try to get first episode of any season - cannot hardcode season or episode number - // because sometimes user can have it in other season/ep than s01e01 - private async processHamaMovie( - metadata: PlexMetadata, - tmdbMovie: TmdbMovieDetails | undefined, - tmdbMovieId: number - ) { - const season = metadata.Children?.Metadata[0]; - if (season) { - const episodes = await this.plexClient.getChildrenMetadata( - season.ratingKey - ); - if (episodes) { - await this.processMovieWithId(episodes[0], tmdbMovie, tmdbMovieId); - } - } - } - - private async processShow(plexitem: PlexLibraryItem) { - const mediaRepository = getRepository(Media); - - let tvShow: TmdbTvDetails | null = null; - - try { - const ratingKey = - plexitem.grandparentRatingKey ?? - plexitem.parentRatingKey ?? - plexitem.ratingKey; - const metadata = await this.plexClient.getMetadata(ratingKey, { - includeChildren: true, - }); - - if (metadata.guid.match(tvdbRegex)) { - const matchedtvdb = metadata.guid.match(tvdbRegex); - - // If we can find a tvdb Id, use it to get the full tmdb show details - if (matchedtvdb?.[1]) { - tvShow = await this.tmdb.getShowByTvdbId({ - tvdbId: Number(matchedtvdb[1]), - }); - } - } else if (metadata.guid.match(tmdbShowRegex)) { - const matchedtmdb = metadata.guid.match(tmdbShowRegex); - - if (matchedtmdb?.[1]) { - tvShow = await this.tmdb.getTvShow({ tvId: Number(matchedtmdb[1]) }); - } - } else if (metadata.guid.match(hamaTvdbRegex)) { - const matched = metadata.guid.match(hamaTvdbRegex); - const tvdbId = matched?.[1]; - - if (tvdbId) { - tvShow = await this.tmdb.getShowByTvdbId({ tvdbId: Number(tvdbId) }); - if (animeList.isLoaded()) { - await this.processHamaSpecials(metadata, Number(tvdbId)); - } else { - this.log( - `Hama ID ${plexitem.guid} detected, but library agent is not set to Hama`, - 'warn' - ); - } - } - } else if (metadata.guid.match(hamaAnidbRegex)) { - const matched = metadata.guid.match(hamaAnidbRegex); - - if (!animeList.isLoaded()) { - this.log( - `Hama ID ${plexitem.guid} detected, but library agent is not set to Hama`, - 'warn' - ); - } else if (matched?.[1]) { - const anidbId = Number(matched[1]); - const result = animeList.getFromAnidbId(anidbId); - - // first try to lookup tvshow by tvdbid - if (result?.tvdbId) { - const extResponse = await this.tmdb.getByExternalId({ - externalId: result.tvdbId, - type: 'tvdb', - }); - if (extResponse.tv_results[0]) { - tvShow = await this.tmdb.getTvShow({ - tvId: extResponse.tv_results[0].id, - }); - } else { - this.log( - `Missing TVDB ${result.tvdbId} entry in TMDB for AniDB ${anidbId}` - ); - } - await this.processHamaSpecials(metadata, result.tvdbId); - } - - if (!tvShow) { - // if lookup of tvshow above failed, then try movie with tmdbid/imdbid - // note - some tv shows have imdbid set too, that's why this need to go second - if (result?.tmdbId) { - return await this.processHamaMovie( - metadata, - undefined, - result.tmdbId - ); - } else if (result?.imdbId) { - const tmdbMovie = await this.tmdb.getMovieByImdbId({ - imdbId: result.imdbId, - }); - return await this.processHamaMovie( - metadata, - tmdbMovie, - tmdbMovie.id - ); - } - } - } - } - - if (tvShow) { - await this.asyncLock.dispatch(tvShow.id, async () => { - if (!tvShow) { - // this will never execute, but typescript thinks somebody could reset tvShow from - // outer scope back to null before this async gets called - return; - } - - // Lets get the available seasons from Plex - const seasons = tvShow.seasons; - const media = await this.getExisting(tvShow.id, MediaType.TV); - - const newSeasons: Season[] = []; - - const currentStandardSeasonAvailable = ( - media?.seasons.filter( - (season) => season.status === MediaStatus.AVAILABLE - ) ?? [] - ).length; - const current4kSeasonAvailable = ( - media?.seasons.filter( - (season) => season.status4k === MediaStatus.AVAILABLE - ) ?? [] - ).length; - - for (const season of seasons) { - const matchedPlexSeason = metadata.Children?.Metadata.find( - (md) => Number(md.index) === season.season_number - ); - - const existingSeason = media?.seasons.find( - (es) => es.seasonNumber === season.season_number - ); - - // Check if we found the matching season and it has all the available episodes - if (matchedPlexSeason) { - // If we have a matched Plex season, get its children metadata so we can check details - const episodes = await this.plexClient.getChildrenMetadata( - matchedPlexSeason.ratingKey - ); - // Total episodes that are in standard definition (not 4k) - const totalStandard = episodes.filter((episode) => - !this.enable4kShow - ? true - : episode.Media.some( - (media) => media.videoResolution !== '4k' - ) - ).length; - - // Total episodes that are in 4k - const total4k = episodes.filter((episode) => - episode.Media.some((media) => media.videoResolution === '4k') - ).length; - - if ( - media && - (totalStandard > 0 || (total4k > 0 && !this.enable4kShow)) && - media.ratingKey !== ratingKey - ) { - media.ratingKey = ratingKey; - } - - if ( - media && - total4k > 0 && - this.enable4kShow && - media.ratingKey4k !== ratingKey - ) { - media.ratingKey4k = ratingKey; - } - - if (existingSeason) { - // These ternary statements look super confusing, but they are simply - // setting the status to AVAILABLE if all of a type is there, partially if some, - // and then not modifying the status if there are 0 items. - // If the season was already available, we don't modify it as well. - existingSeason.status = - totalStandard === season.episode_count || - existingSeason.status === MediaStatus.AVAILABLE - ? MediaStatus.AVAILABLE - : totalStandard > 0 - ? MediaStatus.PARTIALLY_AVAILABLE - : existingSeason.status; - existingSeason.status4k = - (this.enable4kShow && total4k === season.episode_count) || - existingSeason.status4k === MediaStatus.AVAILABLE - ? MediaStatus.AVAILABLE - : this.enable4kShow && total4k > 0 - ? MediaStatus.PARTIALLY_AVAILABLE - : existingSeason.status4k; - } else { - newSeasons.push( - new Season({ - seasonNumber: season.season_number, - // This ternary is the same as the ones above, but it just falls back to "UNKNOWN" - // if we dont have any items for the season - status: - totalStandard === season.episode_count - ? MediaStatus.AVAILABLE - : totalStandard > 0 - ? MediaStatus.PARTIALLY_AVAILABLE - : MediaStatus.UNKNOWN, - status4k: - this.enable4kShow && total4k === season.episode_count - ? MediaStatus.AVAILABLE - : this.enable4kShow && total4k > 0 - ? MediaStatus.PARTIALLY_AVAILABLE - : MediaStatus.UNKNOWN, - }) - ); - } - } - } - - // Remove extras season. We dont count it for determining availability - const filteredSeasons = tvShow.seasons.filter( - (season) => season.season_number !== 0 - ); - - const isAllStandardSeasons = - newSeasons.filter( - (season) => season.status === MediaStatus.AVAILABLE - ).length + - (media?.seasons.filter( - (season) => season.status === MediaStatus.AVAILABLE - ).length ?? 0) >= - filteredSeasons.length; - - const isAll4kSeasons = - newSeasons.filter( - (season) => season.status4k === MediaStatus.AVAILABLE - ).length + - (media?.seasons.filter( - (season) => season.status4k === MediaStatus.AVAILABLE - ).length ?? 0) >= - filteredSeasons.length; - - if (media) { - // Update existing - media.seasons = [...media.seasons, ...newSeasons]; - - const newStandardSeasonAvailable = ( - media.seasons.filter( - (season) => season.status === MediaStatus.AVAILABLE - ) ?? [] - ).length; - - const new4kSeasonAvailable = ( - media.seasons.filter( - (season) => season.status4k === MediaStatus.AVAILABLE - ) ?? [] - ).length; - - // If at least one new season has become available, update - // the lastSeasonChange field so we can trigger notifications - if (newStandardSeasonAvailable > currentStandardSeasonAvailable) { - this.log( - `Detected ${ - newStandardSeasonAvailable - currentStandardSeasonAvailable - } new standard season(s) for ${tvShow.name}`, - 'debug' - ); - media.lastSeasonChange = new Date(); - media.mediaAddedAt = new Date(plexitem.addedAt * 1000); - } - - if (new4kSeasonAvailable > current4kSeasonAvailable) { - this.log( - `Detected ${ - new4kSeasonAvailable - current4kSeasonAvailable - } new 4K season(s) for ${tvShow.name}`, - 'debug' - ); - media.lastSeasonChange = new Date(); - } - - if (!media.mediaAddedAt) { - media.mediaAddedAt = new Date(plexitem.addedAt * 1000); - } - - // If the show is already available, and there are no new seasons, dont adjust - // the status - const shouldStayAvailable = - media.status === MediaStatus.AVAILABLE && - newSeasons.filter( - (season) => season.status !== MediaStatus.UNKNOWN - ).length === 0; - const shouldStayAvailable4k = - media.status4k === MediaStatus.AVAILABLE && - newSeasons.filter( - (season) => season.status4k !== MediaStatus.UNKNOWN - ).length === 0; - - media.status = - isAllStandardSeasons || shouldStayAvailable - ? MediaStatus.AVAILABLE - : media.seasons.some( - (season) => - season.status === MediaStatus.PARTIALLY_AVAILABLE || - season.status === MediaStatus.AVAILABLE - ) - ? MediaStatus.PARTIALLY_AVAILABLE - : MediaStatus.UNKNOWN; - media.status4k = - (isAll4kSeasons || shouldStayAvailable4k) && this.enable4kShow - ? MediaStatus.AVAILABLE - : this.enable4kShow && - media.seasons.some( - (season) => - season.status4k === MediaStatus.PARTIALLY_AVAILABLE || - season.status4k === MediaStatus.AVAILABLE - ) - ? MediaStatus.PARTIALLY_AVAILABLE - : MediaStatus.UNKNOWN; - await mediaRepository.save(media); - this.log(`Updating existing title: ${tvShow.name}`); - } else { - const newMedia = new Media({ - mediaType: MediaType.TV, - seasons: newSeasons, - tmdbId: tvShow.id, - tvdbId: tvShow.external_ids.tvdb_id, - mediaAddedAt: new Date(plexitem.addedAt * 1000), - status: isAllStandardSeasons - ? MediaStatus.AVAILABLE - : newSeasons.some( - (season) => - season.status === MediaStatus.PARTIALLY_AVAILABLE || - season.status === MediaStatus.AVAILABLE - ) - ? MediaStatus.PARTIALLY_AVAILABLE - : MediaStatus.UNKNOWN, - status4k: - isAll4kSeasons && this.enable4kShow - ? MediaStatus.AVAILABLE - : this.enable4kShow && - newSeasons.some( - (season) => - season.status4k === MediaStatus.PARTIALLY_AVAILABLE || - season.status4k === MediaStatus.AVAILABLE - ) - ? MediaStatus.PARTIALLY_AVAILABLE - : MediaStatus.UNKNOWN, - }); - await mediaRepository.save(newMedia); - this.log(`Saved ${tvShow.name}`); - } - }); - } else { - this.log(`failed show: ${plexitem.guid}`); - } - } catch (e) { - this.log( - `Failed to process Plex item. ratingKey: ${ - plexitem.grandparentRatingKey ?? - plexitem.parentRatingKey ?? - plexitem.ratingKey - }`, - 'error', - { - errorMessage: e.message, - plexitem, - } - ); - } - } - - private async processItems(slicedItems: PlexLibraryItem[]) { - await Promise.all( - slicedItems.map(async (plexitem) => { - if (plexitem.type === 'movie') { - await this.processMovie(plexitem); - } else if ( - plexitem.type === 'show' || - plexitem.type === 'episode' || - plexitem.type === 'season' - ) { - await this.processShow(plexitem); - } - }) - ); - } - - private async loop({ - start = 0, - end = BUNDLE_SIZE, - sessionId, - }: { - start?: number; - end?: number; - sessionId?: string; - } = {}) { - const slicedItems = this.items.slice(start, end); - - if (!this.running) { - throw new Error('Sync was aborted.'); - } - - if (this.sessionId !== sessionId) { - throw new Error('New session was started. Old session aborted.'); - } - - if (start < this.items.length) { - this.progress = start; - await this.processItems(slicedItems); - - await new Promise((resolve, reject) => - setTimeout(() => { - this.loop({ - start: start + BUNDLE_SIZE, - end: end + BUNDLE_SIZE, - sessionId, - }) - .then(() => resolve()) - .catch((e) => reject(new Error(e.message))); - }, UPDATE_RATE) - ); - } - } - - private log( - message: string, - level: 'info' | 'error' | 'debug' | 'warn' = 'debug', - optional?: Record - ): void { - logger[level](message, { label: 'Plex Scan', ...optional }); - } - - // checks if any of this.libraries has Hama agent set in Plex - private async hasHamaAgent() { - const plexLibraries = await this.plexClient.getLibraries(); - return this.libraries.some((library) => - plexLibraries.some( - (plexLibrary) => - plexLibrary.agent === HAMA_AGENT && library.id === plexLibrary.key - ) - ); - } - - public async run(): Promise { - const settings = getSettings(); - const sessionId = uuid(); - this.sessionId = sessionId; - logger.info('Plex scan starting', { sessionId, label: 'Plex Scan' }); - try { - this.running = true; - const userRepository = getRepository(User); - const admin = await userRepository.findOne({ - select: ['id', 'plexToken'], - order: { id: 'ASC' }, - }); - - if (!admin) { - return this.log('No admin configured. Plex scan skipped.', 'warn'); - } - - this.plexClient = new PlexAPI({ plexToken: admin.plexToken }); - - this.libraries = settings.plex.libraries.filter( - (library) => library.enabled - ); - - this.enable4kMovie = settings.radarr.some((radarr) => radarr.is4k); - if (this.enable4kMovie) { - this.log( - 'At least one 4K Radarr server was detected. 4K movie detection is now enabled', - 'info' - ); - } - - this.enable4kShow = settings.sonarr.some((sonarr) => sonarr.is4k); - if (this.enable4kShow) { - this.log( - 'At least one 4K Sonarr server was detected. 4K series detection is now enabled', - 'info' - ); - } - - const hasHama = await this.hasHamaAgent(); - if (hasHama) { - await animeList.sync(); - } - - if (this.isRecentOnly) { - for (const library of this.libraries) { - this.currentLibrary = library; - this.log( - `Beginning to process recently added for library: ${library.name}`, - 'info' - ); - const libraryItems = await this.plexClient.getRecentlyAdded( - library.id - ); - - // Bundle items up by rating keys - this.items = uniqWith(libraryItems, (mediaA, mediaB) => { - if (mediaA.grandparentRatingKey && mediaB.grandparentRatingKey) { - return ( - mediaA.grandparentRatingKey === mediaB.grandparentRatingKey - ); - } - - if (mediaA.parentRatingKey && mediaB.parentRatingKey) { - return mediaA.parentRatingKey === mediaB.parentRatingKey; - } - - return mediaA.ratingKey === mediaB.ratingKey; - }); - - await this.loop({ sessionId }); - } - } else { - for (const library of this.libraries) { - this.currentLibrary = library; - this.log(`Beginning to process library: ${library.name}`, 'info'); - this.items = await this.plexClient.getLibraryContents(library.id); - await this.loop({ sessionId }); - } - } - this.log( - this.isRecentOnly - ? 'Recently Added Scan Complete' - : 'Full Scan Complete', - 'info' - ); - } catch (e) { - logger.error('Sync interrupted', { - label: 'Plex Scan', - errorMessage: e.message, - }); - } finally { - // If a new scanning session hasnt started, set running back to false - if (this.sessionId === sessionId) { - this.running = false; - } - } - } - - public status(): SyncStatus { - return { - running: this.running, - progress: this.progress, - total: this.items.length, - currentLibrary: this.currentLibrary, - libraries: this.libraries, - }; - } - - public cancel(): void { - this.running = false; - } -} - -export const jobPlexFullSync = new JobPlexSync(); -export const jobPlexRecentSync = new JobPlexSync({ isRecentOnly: true }); diff --git a/server/job/radarrsync/index.ts b/server/job/radarrsync/index.ts deleted file mode 100644 index e8b0c890..00000000 --- a/server/job/radarrsync/index.ts +++ /dev/null @@ -1,248 +0,0 @@ -import { uniqWith } from 'lodash'; -import { getRepository } from 'typeorm'; -import { v4 as uuid } from 'uuid'; -import RadarrAPI, { RadarrMovie } from '../../api/radarr'; -import { MediaStatus, MediaType } from '../../constants/media'; -import Media from '../../entity/Media'; -import { getSettings, RadarrSettings } from '../../lib/settings'; -import logger from '../../logger'; - -const BUNDLE_SIZE = 50; -const UPDATE_RATE = 4 * 1000; - -interface SyncStatus { - running: boolean; - progress: number; - total: number; - currentServer: RadarrSettings; - servers: RadarrSettings[]; -} - -class JobRadarrSync { - private running = false; - private progress = 0; - private enable4k = false; - private sessionId: string; - private servers: RadarrSettings[]; - private currentServer: RadarrSettings; - private radarrApi: RadarrAPI; - private items: RadarrMovie[] = []; - - public async run() { - const settings = getSettings(); - const sessionId = uuid(); - this.sessionId = sessionId; - this.log('Radarr scan starting', 'info', { sessionId }); - - try { - this.running = true; - - // Remove any duplicate Radarr servers and assign them to the servers field - this.servers = uniqWith(settings.radarr, (radarrA, radarrB) => { - return ( - radarrA.hostname === radarrB.hostname && - radarrA.port === radarrB.port && - radarrA.baseUrl === radarrB.baseUrl - ); - }); - - this.enable4k = settings.radarr.some((radarr) => radarr.is4k); - if (this.enable4k) { - this.log( - 'At least one 4K Radarr server was detected. 4K movie detection is now enabled.', - 'info' - ); - } - - for (const server of this.servers) { - this.currentServer = server; - if (server.syncEnabled) { - this.log( - `Beginning to process Radarr server: ${server.name}`, - 'info' - ); - - this.radarrApi = new RadarrAPI({ - apiKey: server.apiKey, - url: RadarrAPI.buildRadarrUrl(server, '/api/v3'), - }); - - this.items = await this.radarrApi.getMovies(); - - await this.loop({ sessionId }); - } else { - this.log(`Sync not enabled. Skipping Radarr server: ${server.name}`); - } - } - - this.log('Radarr scan complete', 'info'); - } catch (e) { - this.log('Something went wrong.', 'error', { errorMessage: e.message }); - } finally { - // If a new scanning session hasnt started, set running back to false - if (this.sessionId === sessionId) { - this.running = false; - } - } - } - - public status(): SyncStatus { - return { - running: this.running, - progress: this.progress, - total: this.items.length, - currentServer: this.currentServer, - servers: this.servers, - }; - } - - public cancel(): void { - this.running = false; - } - - private async processRadarrMovie(radarrMovie: RadarrMovie) { - const mediaRepository = getRepository(Media); - const server4k = this.enable4k && this.currentServer.is4k; - - const media = await mediaRepository.findOne({ - where: { tmdbId: radarrMovie.tmdbId }, - }); - - if (media) { - let isChanged = false; - if (media.status === MediaStatus.AVAILABLE) { - this.log(`Movie already available: ${radarrMovie.title}`); - } else { - media[server4k ? 'status4k' : 'status'] = radarrMovie.downloaded - ? MediaStatus.AVAILABLE - : MediaStatus.PROCESSING; - this.log( - `Updated existing ${server4k ? '4K ' : ''}movie ${ - radarrMovie.title - } to status ${MediaStatus[media[server4k ? 'status4k' : 'status']]}` - ); - isChanged = true; - } - - if ( - media[server4k ? 'serviceId4k' : 'serviceId'] !== this.currentServer.id - ) { - media[server4k ? 'serviceId4k' : 'serviceId'] = this.currentServer.id; - this.log(`Updated service ID for media entity: ${radarrMovie.title}`); - isChanged = true; - } - - if ( - media[server4k ? 'externalServiceId4k' : 'externalServiceId'] !== - radarrMovie.id - ) { - media[server4k ? 'externalServiceId4k' : 'externalServiceId'] = - radarrMovie.id; - this.log( - `Updated external service ID for media entity: ${radarrMovie.title}` - ); - isChanged = true; - } - - if ( - media[server4k ? 'externalServiceSlug4k' : 'externalServiceSlug'] !== - radarrMovie.titleSlug - ) { - media[server4k ? 'externalServiceSlug4k' : 'externalServiceSlug'] = - radarrMovie.titleSlug; - this.log( - `Updated external service slug for media entity: ${radarrMovie.title}` - ); - isChanged = true; - } - - if (isChanged) { - await mediaRepository.save(media); - } - } else { - const newMedia = new Media({ - tmdbId: radarrMovie.tmdbId, - imdbId: radarrMovie.imdbId, - mediaType: MediaType.MOVIE, - serviceId: !server4k ? this.currentServer.id : undefined, - serviceId4k: server4k ? this.currentServer.id : undefined, - externalServiceId: !server4k ? radarrMovie.id : undefined, - externalServiceId4k: server4k ? radarrMovie.id : undefined, - status: - !server4k && radarrMovie.downloaded - ? MediaStatus.AVAILABLE - : !server4k - ? MediaStatus.PROCESSING - : MediaStatus.UNKNOWN, - status4k: - server4k && radarrMovie.downloaded - ? MediaStatus.AVAILABLE - : server4k - ? MediaStatus.PROCESSING - : MediaStatus.UNKNOWN, - }); - - this.log( - `Added media for movie ${radarrMovie.title} and set status to ${ - MediaStatus[newMedia[server4k ? 'status4k' : 'status']] - }` - ); - await mediaRepository.save(newMedia); - } - } - - private async processItems(items: RadarrMovie[]) { - await Promise.all( - items.map(async (radarrMovie) => { - await this.processRadarrMovie(radarrMovie); - }) - ); - } - - private async loop({ - start = 0, - end = BUNDLE_SIZE, - sessionId, - }: { - start?: number; - end?: number; - sessionId?: string; - } = {}) { - const slicedItems = this.items.slice(start, end); - - if (!this.running) { - throw new Error('Sync was aborted.'); - } - - if (this.sessionId !== sessionId) { - throw new Error('New session was started. Old session aborted.'); - } - - if (start < this.items.length) { - this.progress = start; - await this.processItems(slicedItems); - - await new Promise((resolve, reject) => - setTimeout(() => { - this.loop({ - start: start + BUNDLE_SIZE, - end: end + BUNDLE_SIZE, - sessionId, - }) - .then(() => resolve()) - .catch((e) => reject(new Error(e.message))); - }, UPDATE_RATE) - ); - } - } - - private log( - message: string, - level: 'info' | 'error' | 'debug' | 'warn' = 'debug', - optional?: Record - ): void { - logger[level](message, { label: 'Radarr Scan', ...optional }); - } -} - -export const jobRadarrSync = new JobRadarrSync(); diff --git a/server/job/schedule.ts b/server/job/schedule.ts index 7bbf580d..1e3665b8 100644 --- a/server/job/schedule.ts +++ b/server/job/schedule.ts @@ -1,9 +1,9 @@ import schedule from 'node-schedule'; -import { jobPlexFullSync, jobPlexRecentSync } from './plexsync'; import logger from '../logger'; -import { jobRadarrSync } from './radarrsync'; -import { jobSonarrSync } from './sonarrsync'; import downloadTracker from '../lib/downloadtracker'; +import { plexFullScanner, plexRecentScanner } from '../lib/scanners/plex'; +import { radarrScanner } from '../lib/scanners/radarr'; +import { sonarrScanner } from '../lib/scanners/sonarr'; interface ScheduledJob { id: string; @@ -26,10 +26,10 @@ export const startJobs = (): void => { logger.info('Starting scheduled job: Plex Recently Added Scan', { label: 'Jobs', }); - jobPlexRecentSync.run(); + plexRecentScanner.run(); }), - running: () => jobPlexRecentSync.status().running, - cancelFn: () => jobPlexRecentSync.cancel(), + running: () => plexRecentScanner.status().running, + cancelFn: () => plexRecentScanner.cancel(), }); // Run full plex scan every 24 hours @@ -41,10 +41,10 @@ export const startJobs = (): void => { logger.info('Starting scheduled job: Plex Full Library Scan', { label: 'Jobs', }); - jobPlexFullSync.run(); + plexFullScanner.run(); }), - running: () => jobPlexFullSync.status().running, - cancelFn: () => jobPlexFullSync.cancel(), + running: () => plexFullScanner.status().running, + cancelFn: () => plexFullScanner.cancel(), }); // Run full radarr scan every 24 hours @@ -54,10 +54,10 @@ export const startJobs = (): void => { type: 'process', job: schedule.scheduleJob('0 0 4 * * *', () => { logger.info('Starting scheduled job: Radarr Scan', { label: 'Jobs' }); - jobRadarrSync.run(); + radarrScanner.run(); }), - running: () => jobRadarrSync.status().running, - cancelFn: () => jobRadarrSync.cancel(), + running: () => radarrScanner.status().running, + cancelFn: () => radarrScanner.cancel(), }); // Run full sonarr scan every 24 hours @@ -67,10 +67,10 @@ export const startJobs = (): void => { type: 'process', job: schedule.scheduleJob('0 30 4 * * *', () => { logger.info('Starting scheduled job: Sonarr Scan', { label: 'Jobs' }); - jobSonarrSync.run(); + sonarrScanner.run(); }), - running: () => jobSonarrSync.status().running, - cancelFn: () => jobSonarrSync.cancel(), + running: () => sonarrScanner.status().running, + cancelFn: () => sonarrScanner.cancel(), }); // Run download sync diff --git a/server/job/sonarrsync/index.ts b/server/job/sonarrsync/index.ts deleted file mode 100644 index affcdbb4..00000000 --- a/server/job/sonarrsync/index.ts +++ /dev/null @@ -1,381 +0,0 @@ -import { uniqWith } from 'lodash'; -import { getRepository } from 'typeorm'; -import { v4 as uuid } from 'uuid'; -import SonarrAPI, { SonarrSeries } from '../../api/sonarr'; -import TheMovieDb from '../../api/themoviedb'; -import { TmdbTvDetails } from '../../api/themoviedb/interfaces'; -import { MediaStatus, MediaType } from '../../constants/media'; -import Media from '../../entity/Media'; -import Season from '../../entity/Season'; -import { getSettings, SonarrSettings } from '../../lib/settings'; -import logger from '../../logger'; - -const BUNDLE_SIZE = 50; -const UPDATE_RATE = 4 * 1000; - -interface SyncStatus { - running: boolean; - progress: number; - total: number; - currentServer: SonarrSettings; - servers: SonarrSettings[]; -} - -class JobSonarrSync { - private running = false; - private progress = 0; - private enable4k = false; - private sessionId: string; - private servers: SonarrSettings[]; - private currentServer: SonarrSettings; - private sonarrApi: SonarrAPI; - private items: SonarrSeries[] = []; - - public async run() { - const settings = getSettings(); - const sessionId = uuid(); - this.sessionId = sessionId; - this.log('Sonarr scan starting', 'info', { sessionId }); - - try { - this.running = true; - - // Remove any duplicate Sonarr servers and assign them to the servers field - this.servers = uniqWith(settings.sonarr, (sonarrA, sonarrB) => { - return ( - sonarrA.hostname === sonarrB.hostname && - sonarrA.port === sonarrB.port && - sonarrA.baseUrl === sonarrB.baseUrl - ); - }); - - this.enable4k = settings.sonarr.some((sonarr) => sonarr.is4k); - if (this.enable4k) { - this.log( - 'At least one 4K Sonarr server was detected. 4K movie detection is now enabled.', - 'info' - ); - } - - for (const server of this.servers) { - this.currentServer = server; - if (server.syncEnabled) { - this.log( - `Beginning to process Sonarr server: ${server.name}`, - 'info' - ); - - this.sonarrApi = new SonarrAPI({ - apiKey: server.apiKey, - url: SonarrAPI.buildSonarrUrl(server, '/api/v3'), - }); - - this.items = await this.sonarrApi.getSeries(); - - await this.loop({ sessionId }); - } else { - this.log(`Sync not enabled. Skipping Sonarr server: ${server.name}`); - } - } - - this.log('Sonarr scan complete', 'info'); - } catch (e) { - this.log('Something went wrong.', 'error', { errorMessage: e.message }); - } finally { - // If a new scanning session hasnt started, set running back to false - if (this.sessionId === sessionId) { - this.running = false; - } - } - } - - public status(): SyncStatus { - return { - running: this.running, - progress: this.progress, - total: this.items.length, - currentServer: this.currentServer, - servers: this.servers, - }; - } - - public cancel(): void { - this.running = false; - } - - private async processSonarrSeries(sonarrSeries: SonarrSeries) { - const mediaRepository = getRepository(Media); - const server4k = this.enable4k && this.currentServer.is4k; - - const media = await mediaRepository.findOne({ - where: { tvdbId: sonarrSeries.tvdbId }, - }); - - const currentSeasonsAvailable = (media?.seasons ?? []).filter( - (season) => - season[server4k ? 'status4k' : 'status'] === MediaStatus.AVAILABLE - ).length; - - const newSeasons: Season[] = []; - - for (const season of sonarrSeries.seasons) { - const existingSeason = media?.seasons.find( - (es) => es.seasonNumber === season.seasonNumber - ); - - // We are already tracking this season so we can work on it directly - if (existingSeason) { - if ( - existingSeason[server4k ? 'status4k' : 'status'] !== - MediaStatus.AVAILABLE && - season.statistics - ) { - existingSeason[server4k ? 'status4k' : 'status'] = - season.statistics.episodeFileCount === - season.statistics.totalEpisodeCount - ? MediaStatus.AVAILABLE - : season.statistics.episodeFileCount > 0 - ? MediaStatus.PARTIALLY_AVAILABLE - : season.monitored - ? MediaStatus.PROCESSING - : existingSeason[server4k ? 'status4k' : 'status']; - } - } else { - if (season.statistics && season.seasonNumber !== 0) { - const allEpisodes = - season.statistics.episodeFileCount === - season.statistics.totalEpisodeCount; - newSeasons.push( - new Season({ - seasonNumber: season.seasonNumber, - status: - !server4k && allEpisodes - ? MediaStatus.AVAILABLE - : !server4k && season.statistics.episodeFileCount > 0 - ? MediaStatus.PARTIALLY_AVAILABLE - : !server4k && season.monitored - ? MediaStatus.PROCESSING - : MediaStatus.UNKNOWN, - status4k: - server4k && allEpisodes - ? MediaStatus.AVAILABLE - : server4k && season.statistics.episodeFileCount > 0 - ? MediaStatus.PARTIALLY_AVAILABLE - : !server4k && season.monitored - ? MediaStatus.PROCESSING - : MediaStatus.UNKNOWN, - }) - ); - } - } - } - - const filteredSeasons = sonarrSeries.seasons.filter( - (s) => s.seasonNumber !== 0 - ); - - const isAllSeasons = - (media?.seasons ?? []).filter( - (s) => s[server4k ? 'status4k' : 'status'] === MediaStatus.AVAILABLE - ).length + - newSeasons.filter( - (s) => s[server4k ? 'status4k' : 'status'] === MediaStatus.AVAILABLE - ).length >= - filteredSeasons.length && filteredSeasons.length > 0; - - if (media) { - media.seasons = [...media.seasons, ...newSeasons]; - - const newSeasonsAvailable = (media?.seasons ?? []).filter( - (season) => - season[server4k ? 'status4k' : 'status'] === MediaStatus.AVAILABLE - ).length; - - if (newSeasonsAvailable > currentSeasonsAvailable) { - this.log( - `Detected ${newSeasonsAvailable - currentSeasonsAvailable} new ${ - server4k ? '4K ' : '' - }season(s) for ${sonarrSeries.title}`, - 'debug' - ); - media.lastSeasonChange = new Date(); - } - - if ( - media[server4k ? 'serviceId4k' : 'serviceId'] !== this.currentServer.id - ) { - media[server4k ? 'serviceId4k' : 'serviceId'] = this.currentServer.id; - this.log(`Updated service ID for media entity: ${sonarrSeries.title}`); - } - - if ( - media[server4k ? 'externalServiceId4k' : 'externalServiceId'] !== - sonarrSeries.id - ) { - media[server4k ? 'externalServiceId4k' : 'externalServiceId'] = - sonarrSeries.id; - this.log( - `Updated external service ID for media entity: ${sonarrSeries.title}` - ); - } - - if ( - media[server4k ? 'externalServiceSlug4k' : 'externalServiceSlug'] !== - sonarrSeries.titleSlug - ) { - media[server4k ? 'externalServiceSlug4k' : 'externalServiceSlug'] = - sonarrSeries.titleSlug; - this.log( - `Updated external service slug for media entity: ${sonarrSeries.title}` - ); - } - - // If the show is already available, and there are no new seasons, dont adjust - // the status - const shouldStayAvailable = - media.status === MediaStatus.AVAILABLE && - newSeasons.filter( - (season) => - season[server4k ? 'status4k' : 'status'] !== MediaStatus.UNKNOWN - ).length === 0; - - media[server4k ? 'status4k' : 'status'] = - isAllSeasons || shouldStayAvailable - ? MediaStatus.AVAILABLE - : media.seasons.some( - (season) => - season[server4k ? 'status4k' : 'status'] === - MediaStatus.AVAILABLE || - season[server4k ? 'status4k' : 'status'] === - MediaStatus.PARTIALLY_AVAILABLE - ) - ? MediaStatus.PARTIALLY_AVAILABLE - : media.seasons.some( - (season) => - season[server4k ? 'status4k' : 'status'] === - MediaStatus.PROCESSING - ) - ? MediaStatus.PROCESSING - : MediaStatus.UNKNOWN; - - await mediaRepository.save(media); - } else { - const tmdb = new TheMovieDb(); - let tvShow: TmdbTvDetails; - - try { - tvShow = await tmdb.getShowByTvdbId({ - tvdbId: sonarrSeries.tvdbId, - }); - } catch (e) { - this.log( - 'Failed to create new media item during sync. TVDB ID is missing from TMDB?', - 'warn', - { sonarrSeries, errorMessage: e.message } - ); - return; - } - - const newMedia = new Media({ - tmdbId: tvShow.id, - tvdbId: sonarrSeries.tvdbId, - mediaType: MediaType.TV, - serviceId: !server4k ? this.currentServer.id : undefined, - serviceId4k: server4k ? this.currentServer.id : undefined, - externalServiceId: !server4k ? sonarrSeries.id : undefined, - externalServiceId4k: server4k ? sonarrSeries.id : undefined, - externalServiceSlug: !server4k ? sonarrSeries.titleSlug : undefined, - externalServiceSlug4k: server4k ? sonarrSeries.titleSlug : undefined, - seasons: newSeasons, - status: - !server4k && isAllSeasons - ? MediaStatus.AVAILABLE - : !server4k && - newSeasons.some( - (s) => - s.status === MediaStatus.PARTIALLY_AVAILABLE || - s.status === MediaStatus.AVAILABLE - ) - ? MediaStatus.PARTIALLY_AVAILABLE - : !server4k - ? MediaStatus.PROCESSING - : MediaStatus.UNKNOWN, - status4k: - server4k && isAllSeasons - ? MediaStatus.AVAILABLE - : server4k && - newSeasons.some( - (s) => - s.status4k === MediaStatus.PARTIALLY_AVAILABLE || - s.status4k === MediaStatus.AVAILABLE - ) - ? MediaStatus.PARTIALLY_AVAILABLE - : server4k - ? MediaStatus.PROCESSING - : MediaStatus.UNKNOWN, - }); - - this.log( - `Added media for series ${sonarrSeries.title} and set status to ${ - MediaStatus[newMedia[server4k ? 'status4k' : 'status']] - }` - ); - await mediaRepository.save(newMedia); - } - } - - private async processItems(items: SonarrSeries[]) { - await Promise.all( - items.map(async (sonarrSeries) => { - await this.processSonarrSeries(sonarrSeries); - }) - ); - } - - private async loop({ - start = 0, - end = BUNDLE_SIZE, - sessionId, - }: { - start?: number; - end?: number; - sessionId?: string; - } = {}) { - const slicedItems = this.items.slice(start, end); - - if (!this.running) { - throw new Error('Sync was aborted.'); - } - - if (this.sessionId !== sessionId) { - throw new Error('New session was started. Old session aborted.'); - } - - if (start < this.items.length) { - this.progress = start; - await this.processItems(slicedItems); - - await new Promise((resolve, reject) => - setTimeout(() => { - this.loop({ - start: start + BUNDLE_SIZE, - end: end + BUNDLE_SIZE, - sessionId, - }) - .then(() => resolve()) - .catch((e) => reject(new Error(e.message))); - }, UPDATE_RATE) - ); - } - } - - private log( - message: string, - level: 'info' | 'error' | 'debug' | 'warn' = 'debug', - optional?: Record - ): void { - logger[level](message, { label: 'Sonarr Scan', ...optional }); - } -} - -export const jobSonarrSync = new JobSonarrSync(); diff --git a/server/lib/scanners/baseScanner.ts b/server/lib/scanners/baseScanner.ts new file mode 100644 index 00000000..d845a352 --- /dev/null +++ b/server/lib/scanners/baseScanner.ts @@ -0,0 +1,616 @@ +import { getRepository } from 'typeorm'; +import { v4 as uuid } from 'uuid'; +import TheMovieDb from '../../api/themoviedb'; +import { MediaStatus, MediaType } from '../../constants/media'; +import Media from '../../entity/Media'; +import Season from '../../entity/Season'; +import logger from '../../logger'; +import AsyncLock from '../../utils/asyncLock'; +import { getSettings } from '../settings'; + +// Default scan rates (can be overidden) +const BUNDLE_SIZE = 20; +const UPDATE_RATE = 4 * 1000; + +export type StatusBase = { + running: boolean; + progress: number; + total: number; +}; + +export interface RunnableScanner { + run: () => Promise; + status: () => T & StatusBase; +} + +export interface MediaIds { + tmdbId: number; + imdbId?: string; + tvdbId?: number; + isHama?: boolean; +} + +interface ProcessOptions { + is4k?: boolean; + mediaAddedAt?: Date; + ratingKey?: string; + serviceId?: number; + externalServiceId?: number; + externalServiceSlug?: string; + title?: string; + processing?: boolean; +} + +export interface ProcessableSeason { + seasonNumber: number; + totalEpisodes: number; + episodes: number; + episodes4k: number; + is4kOverride?: boolean; + processing?: boolean; +} + +class BaseScanner { + private bundleSize; + private updateRate; + protected progress = 0; + protected items: T[] = []; + protected scannerName: string; + protected enable4kMovie = false; + protected enable4kShow = false; + protected sessionId: string; + protected running = false; + readonly asyncLock = new AsyncLock(); + readonly tmdb = new TheMovieDb(); + + protected constructor( + scannerName: string, + { + updateRate, + bundleSize, + }: { + updateRate?: number; + bundleSize?: number; + } = {} + ) { + this.scannerName = scannerName; + this.bundleSize = bundleSize ?? BUNDLE_SIZE; + this.updateRate = updateRate ?? UPDATE_RATE; + } + + private async getExisting(tmdbId: number, mediaType: MediaType) { + const mediaRepository = getRepository(Media); + + const existing = await mediaRepository.findOne({ + where: { tmdbId: tmdbId, mediaType }, + }); + + return existing; + } + + protected async processMovie( + tmdbId: number, + { + is4k = false, + mediaAddedAt, + ratingKey, + serviceId, + externalServiceId, + externalServiceSlug, + processing = false, + title = 'Unknown Title', + }: ProcessOptions = {} + ): Promise { + const mediaRepository = getRepository(Media); + + await this.asyncLock.dispatch(tmdbId, async () => { + const existing = await this.getExisting(tmdbId, MediaType.MOVIE); + + if (existing) { + let changedExisting = false; + + if (existing[is4k ? 'status4k' : 'status'] !== MediaStatus.AVAILABLE) { + existing[is4k ? 'status4k' : 'status'] = processing + ? MediaStatus.PROCESSING + : MediaStatus.AVAILABLE; + if (mediaAddedAt) { + existing.mediaAddedAt = mediaAddedAt; + } + changedExisting = true; + } + + if (!changedExisting && !existing.mediaAddedAt && mediaAddedAt) { + existing.mediaAddedAt = mediaAddedAt; + changedExisting = true; + } + + if ( + ratingKey && + existing[is4k ? 'ratingKey4k' : 'ratingKey'] !== ratingKey + ) { + existing[is4k ? 'ratingKey4k' : 'ratingKey'] = ratingKey; + changedExisting = true; + } + + if ( + serviceId !== undefined && + existing[is4k ? 'serviceId4k' : 'serviceId'] !== serviceId + ) { + existing[is4k ? 'serviceId4k' : 'serviceId'] = serviceId; + changedExisting = true; + } + + if ( + externalServiceId !== undefined && + existing[is4k ? 'externalServiceId4k' : 'externalServiceId'] !== + externalServiceId + ) { + existing[ + is4k ? 'externalServiceId4k' : 'externalServiceId' + ] = externalServiceId; + changedExisting = true; + } + + if ( + externalServiceSlug !== undefined && + existing[is4k ? 'externalServiceSlug4k' : 'externalServiceSlug'] !== + externalServiceSlug + ) { + existing[ + is4k ? 'externalServiceSlug4k' : 'externalServiceSlug' + ] = externalServiceSlug; + changedExisting = true; + } + + if (changedExisting) { + await mediaRepository.save(existing); + this.log( + `Media for ${title} exists. Changed were detected and the title will be updated.`, + 'info' + ); + } else { + this.log(`Title already exists and no changes detected for ${title}`); + } + } else { + const newMedia = new Media(); + newMedia.tmdbId = tmdbId; + + newMedia.status = + !is4k && !processing + ? MediaStatus.AVAILABLE + : !is4k && processing + ? MediaStatus.PROCESSING + : MediaStatus.UNKNOWN; + newMedia.status4k = + is4k && this.enable4kMovie && !processing + ? MediaStatus.AVAILABLE + : is4k && this.enable4kMovie && processing + ? MediaStatus.PROCESSING + : MediaStatus.UNKNOWN; + newMedia.mediaType = MediaType.MOVIE; + newMedia.serviceId = !is4k ? serviceId : undefined; + newMedia.serviceId4k = is4k ? serviceId : undefined; + newMedia.externalServiceId = !is4k ? externalServiceId : undefined; + newMedia.externalServiceId4k = is4k ? externalServiceId : undefined; + newMedia.externalServiceSlug = !is4k ? externalServiceSlug : undefined; + newMedia.externalServiceSlug4k = is4k ? externalServiceSlug : undefined; + + if (mediaAddedAt) { + newMedia.mediaAddedAt = mediaAddedAt; + } + + if (ratingKey) { + newMedia.ratingKey = !is4k ? ratingKey : undefined; + newMedia.ratingKey4k = + is4k && this.enable4kMovie ? ratingKey : undefined; + } + await mediaRepository.save(newMedia); + this.log(`Saved new media: ${title}`); + } + }); + } + + /** + * processShow takes a TMDb ID and an array of ProcessableSeasons, which + * should include the total episodes a sesaon has + the total available + * episodes that each season currently has. Unlike processMovie, this method + * does not take an `is4k` option. We handle both the 4k _and_ non 4k status + * in one method. + * + * Note: If 4k is not enable, ProcessableSeasons should combine their episode counts + * into the normal episodes properties and avoid using the 4k properties. + */ + protected async processShow( + tmdbId: number, + tvdbId: number, + seasons: ProcessableSeason[], + { + mediaAddedAt, + ratingKey, + serviceId, + externalServiceId, + externalServiceSlug, + is4k = false, + title = 'Unknown Title', + }: ProcessOptions = {} + ): Promise { + const mediaRepository = getRepository(Media); + + await this.asyncLock.dispatch(tmdbId, async () => { + const media = await this.getExisting(tmdbId, MediaType.TV); + + const newSeasons: Season[] = []; + + const currentStandardSeasonsAvailable = ( + media?.seasons.filter( + (season) => season.status === MediaStatus.AVAILABLE + ) ?? [] + ).length; + + const current4kSeasonsAvailable = ( + media?.seasons.filter( + (season) => season.status4k === MediaStatus.AVAILABLE + ) ?? [] + ).length; + + for (const season of seasons) { + const existingSeason = media?.seasons.find( + (es) => es.seasonNumber === season.seasonNumber + ); + + // We update the rating keys in the seasons loop because we need episode counts + if (media && season.episodes > 0 && media.ratingKey !== ratingKey) { + media.ratingKey = ratingKey; + } + + if ( + media && + season.episodes4k > 0 && + this.enable4kShow && + media.ratingKey4k !== ratingKey + ) { + media.ratingKey4k = ratingKey; + } + + if (existingSeason) { + // Here we update seasons if they already exist. + // If the season is already marked as available, we + // force it to stay available (to avoid competing scanners) + existingSeason.status = + (season.totalEpisodes === season.episodes && season.episodes > 0) || + existingSeason.status === MediaStatus.AVAILABLE + ? MediaStatus.AVAILABLE + : season.episodes > 0 + ? MediaStatus.PARTIALLY_AVAILABLE + : !season.is4kOverride && season.processing + ? MediaStatus.PROCESSING + : existingSeason.status; + + // Same thing here, except we only do updates if 4k is enabled + existingSeason.status4k = + (this.enable4kShow && + season.episodes4k === season.totalEpisodes && + season.episodes4k > 0) || + existingSeason.status4k === MediaStatus.AVAILABLE + ? MediaStatus.AVAILABLE + : this.enable4kShow && season.episodes4k > 0 + ? MediaStatus.PARTIALLY_AVAILABLE + : season.is4kOverride && season.processing + ? MediaStatus.PROCESSING + : existingSeason.status4k; + } else { + newSeasons.push( + new Season({ + seasonNumber: season.seasonNumber, + status: + season.totalEpisodes === season.episodes && season.episodes > 0 + ? MediaStatus.AVAILABLE + : season.episodes > 0 + ? MediaStatus.PARTIALLY_AVAILABLE + : !season.is4kOverride && season.processing + ? MediaStatus.PROCESSING + : MediaStatus.UNKNOWN, + status4k: + this.enable4kShow && + season.totalEpisodes === season.episodes4k && + season.episodes4k > 0 + ? MediaStatus.AVAILABLE + : this.enable4kShow && season.episodes4k > 0 + ? MediaStatus.PARTIALLY_AVAILABLE + : season.is4kOverride && season.processing + ? MediaStatus.PROCESSING + : MediaStatus.UNKNOWN, + }) + ); + } + } + + const isAllStandardSeasons = + seasons.length && + seasons.every( + (season) => + season.episodes === season.totalEpisodes && season.episodes > 0 + ); + + const isAll4kSeasons = + seasons.length && + seasons.every( + (season) => + season.episodes4k === season.totalEpisodes && season.episodes4k > 0 + ); + + if (media) { + media.seasons = [...media.seasons, ...newSeasons]; + + const newStandardSeasonsAvailable = ( + media.seasons.filter( + (season) => season.status === MediaStatus.AVAILABLE + ) ?? [] + ).length; + + const new4kSeasonsAvailable = ( + media.seasons.filter( + (season) => season.status4k === MediaStatus.AVAILABLE + ) ?? [] + ).length; + + // If at least one new season has become available, update + // the lastSeasonChange field so we can trigger notifications + if (newStandardSeasonsAvailable > currentStandardSeasonsAvailable) { + this.log( + `Detected ${ + newStandardSeasonsAvailable - currentStandardSeasonsAvailable + } new standard season(s) for ${title}`, + 'debug' + ); + media.lastSeasonChange = new Date(); + + if (mediaAddedAt) { + media.mediaAddedAt = mediaAddedAt; + } + } + + if (new4kSeasonsAvailable > current4kSeasonsAvailable) { + this.log( + `Detected ${ + new4kSeasonsAvailable - current4kSeasonsAvailable + } new 4K season(s) for ${title}`, + 'debug' + ); + media.lastSeasonChange = new Date(); + } + + if (!media.mediaAddedAt && mediaAddedAt) { + media.mediaAddedAt = mediaAddedAt; + } + + if (serviceId !== undefined) { + media[is4k ? 'serviceId4k' : 'serviceId'] = serviceId; + } + + if (externalServiceId !== undefined) { + media[ + is4k ? 'externalServiceId4k' : 'externalServiceId' + ] = externalServiceId; + } + + if (externalServiceSlug !== undefined) { + media[ + is4k ? 'externalServiceSlug4k' : 'externalServiceSlug' + ] = externalServiceSlug; + } + + // If the show is already available, and there are no new seasons, dont adjust + // the status + const shouldStayAvailable = + media.status === MediaStatus.AVAILABLE && + newSeasons.filter((season) => season.status !== MediaStatus.UNKNOWN) + .length === 0; + const shouldStayAvailable4k = + media.status4k === MediaStatus.AVAILABLE && + newSeasons.filter((season) => season.status4k !== MediaStatus.UNKNOWN) + .length === 0; + + media.status = + isAllStandardSeasons || shouldStayAvailable + ? MediaStatus.AVAILABLE + : media.seasons.some( + (season) => + season.status === MediaStatus.PARTIALLY_AVAILABLE || + season.status === MediaStatus.AVAILABLE + ) + ? MediaStatus.PARTIALLY_AVAILABLE + : media.seasons.some( + (season) => season.status === MediaStatus.PROCESSING + ) + ? MediaStatus.PROCESSING + : MediaStatus.UNKNOWN; + media.status4k = + (isAll4kSeasons || shouldStayAvailable4k) && this.enable4kShow + ? MediaStatus.AVAILABLE + : this.enable4kShow && + media.seasons.some( + (season) => + season.status4k === MediaStatus.PARTIALLY_AVAILABLE || + season.status4k === MediaStatus.AVAILABLE + ) + ? MediaStatus.PARTIALLY_AVAILABLE + : media.seasons.some( + (season) => season.status4k === MediaStatus.PROCESSING + ) + ? MediaStatus.PROCESSING + : MediaStatus.UNKNOWN; + await mediaRepository.save(media); + this.log(`Updating existing title: ${title}`); + } else { + const newMedia = new Media({ + mediaType: MediaType.TV, + seasons: newSeasons, + tmdbId, + tvdbId, + mediaAddedAt, + serviceId: !is4k ? serviceId : undefined, + serviceId4k: is4k ? serviceId : undefined, + externalServiceId: !is4k ? externalServiceId : undefined, + externalServiceId4k: is4k ? externalServiceId : undefined, + externalServiceSlug: !is4k ? externalServiceSlug : undefined, + externalServiceSlug4k: is4k ? externalServiceSlug : undefined, + ratingKey: newSeasons.some( + (sn) => + sn.status === MediaStatus.PARTIALLY_AVAILABLE || + sn.status === MediaStatus.AVAILABLE + ) + ? ratingKey + : undefined, + ratingKey4k: + this.enable4kShow && + newSeasons.some( + (sn) => + sn.status4k === MediaStatus.PARTIALLY_AVAILABLE || + sn.status4k === MediaStatus.AVAILABLE + ) + ? ratingKey + : undefined, + status: isAllStandardSeasons + ? MediaStatus.AVAILABLE + : newSeasons.some( + (season) => + season.status === MediaStatus.PARTIALLY_AVAILABLE || + season.status === MediaStatus.AVAILABLE + ) + ? MediaStatus.PARTIALLY_AVAILABLE + : newSeasons.some( + (season) => season.status === MediaStatus.PROCESSING + ) + ? MediaStatus.PROCESSING + : MediaStatus.UNKNOWN, + status4k: + isAll4kSeasons && this.enable4kShow + ? MediaStatus.AVAILABLE + : this.enable4kShow && + newSeasons.some( + (season) => + season.status4k === MediaStatus.PARTIALLY_AVAILABLE || + season.status4k === MediaStatus.AVAILABLE + ) + ? MediaStatus.PARTIALLY_AVAILABLE + : newSeasons.some( + (season) => season.status4k === MediaStatus.PROCESSING + ) + ? MediaStatus.PROCESSING + : MediaStatus.UNKNOWN, + }); + await mediaRepository.save(newMedia); + this.log(`Saved ${title}`); + } + }); + } + + /** + * Call startRun from child class whenever a run is starting to + * ensure required values are set + * + * Returns the session ID which is requried for the cleanup method + */ + protected startRun(): string { + const settings = getSettings(); + const sessionId = uuid(); + this.sessionId = sessionId; + + this.log('Scan starting', 'info', { sessionId }); + + this.enable4kMovie = settings.radarr.some((radarr) => radarr.is4k); + if (this.enable4kMovie) { + this.log( + 'At least one 4K Radarr server was detected. 4K movie detection is now enabled', + 'info' + ); + } + + this.enable4kShow = settings.sonarr.some((sonarr) => sonarr.is4k); + if (this.enable4kShow) { + this.log( + 'At least one 4K Sonarr server was detected. 4K series detection is now enabled', + 'info' + ); + } + + this.running = true; + + return sessionId; + } + + /** + * Call at end of run loop to perform cleanup + */ + protected endRun(sessionId: string): void { + if (this.sessionId === sessionId) { + this.running = false; + } + } + + public cancel(): void { + this.running = false; + } + + protected async loop( + processFn: (item: T) => Promise, + { + start = 0, + end = this.bundleSize, + sessionId, + }: { + start?: number; + end?: number; + sessionId?: string; + } = {} + ): Promise { + const slicedItems = this.items.slice(start, end); + + if (!this.running) { + throw new Error('Sync was aborted.'); + } + + if (this.sessionId !== sessionId) { + throw new Error('New session was started. Old session aborted.'); + } + + if (start < this.items.length) { + this.progress = start; + await this.processItems(processFn, slicedItems); + + await new Promise((resolve, reject) => + setTimeout(() => { + this.loop(processFn, { + start: start + this.bundleSize, + end: end + this.bundleSize, + sessionId, + }) + .then(() => resolve()) + .catch((e) => reject(new Error(e.message))); + }, this.updateRate) + ); + } + } + + private async processItems( + processFn: (items: T) => Promise, + items: T[] + ) { + await Promise.all( + items.map(async (item) => { + await processFn(item); + }) + ); + } + + protected log( + message: string, + level: 'info' | 'error' | 'debug' | 'warn' = 'debug', + optional?: Record + ): void { + logger[level](message, { label: this.scannerName, ...optional }); + } +} + +export default BaseScanner; diff --git a/server/lib/scanners/plex/index.ts b/server/lib/scanners/plex/index.ts new file mode 100644 index 00000000..dc136900 --- /dev/null +++ b/server/lib/scanners/plex/index.ts @@ -0,0 +1,463 @@ +import { uniqWith } from 'lodash'; +import { getRepository } from 'typeorm'; +import animeList from '../../../api/animelist'; +import PlexAPI, { PlexLibraryItem, PlexMetadata } from '../../../api/plexapi'; +import { TmdbTvDetails } from '../../../api/themoviedb/interfaces'; +import { User } from '../../../entity/User'; +import { getSettings, Library } from '../../settings'; +import BaseScanner, { + MediaIds, + RunnableScanner, + StatusBase, + ProcessableSeason, +} from '../baseScanner'; + +const imdbRegex = new RegExp(/imdb:\/\/(tt[0-9]+)/); +const tmdbRegex = new RegExp(/tmdb:\/\/([0-9]+)/); +const tvdbRegex = new RegExp(/tvdb:\/\/([0-9]+)/); +const tmdbShowRegex = new RegExp(/themoviedb:\/\/([0-9]+)/); +const plexRegex = new RegExp(/plex:\/\//); +// Hama agent uses ASS naming, see details here: +// https://github.com/ZeroQI/Absolute-Series-Scanner/blob/master/README.md#forcing-the-movieseries-id +const hamaTvdbRegex = new RegExp(/hama:\/\/tvdb[0-9]?-([0-9]+)/); +const hamaAnidbRegex = new RegExp(/hama:\/\/anidb[0-9]?-([0-9]+)/); +const HAMA_AGENT = 'com.plexapp.agents.hama'; + +type SyncStatus = StatusBase & { + currentLibrary: Library; + libraries: Library[]; +}; + +class PlexScanner + extends BaseScanner + implements RunnableScanner { + private plexClient: PlexAPI; + private libraries: Library[]; + private currentLibrary: Library; + private isRecentOnly = false; + + public constructor(isRecentOnly = false) { + super('Plex Scan'); + this.isRecentOnly = isRecentOnly; + } + + public status(): SyncStatus { + return { + running: this.running, + progress: this.progress, + total: this.items.length, + currentLibrary: this.currentLibrary, + libraries: this.libraries, + }; + } + + public async run(): Promise { + const settings = getSettings(); + const sessionId = this.startRun(); + try { + const userRepository = getRepository(User); + const admin = await userRepository.findOne({ + select: ['id', 'plexToken'], + order: { id: 'ASC' }, + }); + + if (!admin) { + return this.log('No admin configured. Plex scan skipped.', 'warn'); + } + + this.plexClient = new PlexAPI({ plexToken: admin.plexToken }); + + this.libraries = settings.plex.libraries.filter( + (library) => library.enabled + ); + + const hasHama = await this.hasHamaAgent(); + if (hasHama) { + await animeList.sync(); + } + + if (this.isRecentOnly) { + for (const library of this.libraries) { + this.currentLibrary = library; + this.log( + `Beginning to process recently added for library: ${library.name}`, + 'info' + ); + const libraryItems = await this.plexClient.getRecentlyAdded( + library.id + ); + + // Bundle items up by rating keys + this.items = uniqWith(libraryItems, (mediaA, mediaB) => { + if (mediaA.grandparentRatingKey && mediaB.grandparentRatingKey) { + return ( + mediaA.grandparentRatingKey === mediaB.grandparentRatingKey + ); + } + + if (mediaA.parentRatingKey && mediaB.parentRatingKey) { + return mediaA.parentRatingKey === mediaB.parentRatingKey; + } + + return mediaA.ratingKey === mediaB.ratingKey; + }); + + await this.loop(this.processItem.bind(this), { sessionId }); + } + } else { + for (const library of this.libraries) { + this.currentLibrary = library; + this.log(`Beginning to process library: ${library.name}`, 'info'); + this.items = await this.plexClient.getLibraryContents(library.id); + await this.loop(this.processItem.bind(this), { sessionId }); + } + } + this.log( + this.isRecentOnly + ? 'Recently Added Scan Complete' + : 'Full Scan Complete', + 'info' + ); + } catch (e) { + this.log('Scan interrupted', 'error', { errorMessage: e.message }); + } finally { + this.endRun(sessionId); + } + } + + private async processItem(plexitem: PlexLibraryItem) { + try { + if (plexitem.type === 'movie') { + await this.processPlexMovie(plexitem); + } else if ( + plexitem.type === 'show' || + plexitem.type === 'episode' || + plexitem.type === 'season' + ) { + await this.processPlexShow(plexitem); + } + } catch (e) { + this.log('Failed to process Plex media', 'error', { + errorMessage: e.message, + title: plexitem.title, + }); + } + } + + private async processPlexMovie(plexitem: PlexLibraryItem) { + const mediaIds = await this.getMediaIds(plexitem); + const metadata = await this.plexClient.getMetadata(plexitem.ratingKey); + + const has4k = metadata.Media.some( + (media) => media.videoResolution === '4k' + ); + + await this.processMovie(mediaIds.tmdbId, { + is4k: has4k && this.enable4kMovie, + mediaAddedAt: new Date(plexitem.addedAt * 1000), + ratingKey: plexitem.ratingKey, + title: plexitem.title, + }); + } + + private async processPlexMovieByTmdbId( + plexitem: PlexMetadata, + tmdbId: number + ) { + const has4k = plexitem.Media.some( + (media) => media.videoResolution === '4k' + ); + + await this.processMovie(tmdbId, { + is4k: has4k && this.enable4kMovie, + mediaAddedAt: new Date(plexitem.addedAt * 1000), + ratingKey: plexitem.ratingKey, + title: plexitem.title, + }); + } + + private async processPlexShow(plexitem: PlexLibraryItem) { + const ratingKey = + plexitem.grandparentRatingKey ?? + plexitem.parentRatingKey ?? + plexitem.ratingKey; + const metadata = await this.plexClient.getMetadata(ratingKey, { + includeChildren: true, + }); + + const mediaIds = await this.getMediaIds(metadata); + + // If the media is from HAMA, and doesn't have a TVDb ID, we will treat it + // as a special HAMA movie + if (mediaIds.tmdbId && !mediaIds.tvdbId && mediaIds.isHama) { + this.processHamaMovie(metadata, mediaIds.tmdbId); + return; + } + + // If the media is from HAMA and we have a TVDb ID, we will attempt + // to process any specials that may exist + if (mediaIds.tvdbId && mediaIds.isHama) { + await this.processHamaSpecials(metadata, mediaIds.tvdbId); + } + + const tvShow = await this.tmdb.getTvShow({ tvId: mediaIds.tmdbId }); + + const seasons = tvShow.seasons; + const processableSeasons: ProcessableSeason[] = []; + + const filteredSeasons = seasons.filter((sn) => sn.season_number !== 0); + + for (const season of filteredSeasons) { + const matchedPlexSeason = metadata.Children?.Metadata.find( + (md) => Number(md.index) === season.season_number + ); + + if (matchedPlexSeason) { + // If we have a matched Plex season, get its children metadata so we can check details + const episodes = await this.plexClient.getChildrenMetadata( + matchedPlexSeason.ratingKey + ); + // Total episodes that are in standard definition (not 4k) + const totalStandard = episodes.filter((episode) => + !this.enable4kShow + ? true + : episode.Media.some((media) => media.videoResolution !== '4k') + ).length; + + // Total episodes that are in 4k + const total4k = this.enable4kShow + ? episodes.filter((episode) => + episode.Media.some((media) => media.videoResolution === '4k') + ).length + : 0; + + processableSeasons.push({ + seasonNumber: season.season_number, + episodes: totalStandard, + episodes4k: total4k, + totalEpisodes: season.episode_count, + }); + } else { + processableSeasons.push({ + seasonNumber: season.season_number, + episodes: 0, + episodes4k: 0, + totalEpisodes: season.episode_count, + }); + } + } + + if (mediaIds.tvdbId) { + await this.processShow( + mediaIds.tmdbId, + mediaIds.tvdbId ?? tvShow.external_ids.tvdb_id, + processableSeasons, + { + mediaAddedAt: new Date(metadata.addedAt * 1000), + ratingKey: ratingKey, + title: metadata.title, + } + ); + } + } + + private async getMediaIds(plexitem: PlexLibraryItem): Promise { + const mediaIds: Partial = {}; + // Check if item is using new plex movie/tv agent + if (plexitem.guid.match(plexRegex)) { + const metadata = await this.plexClient.getMetadata(plexitem.ratingKey); + + // If there is no Guid field at all, then we bail + if (!metadata.Guid) { + throw new Error( + 'No Guid metadata for this title. Skipping. (Try refreshing the metadata in Plex for this media!)' + ); + } + + // Map all IDs to MediaId object + metadata.Guid.forEach((ref) => { + if (ref.id.match(imdbRegex)) { + mediaIds.imdbId = ref.id.match(imdbRegex)?.[1] ?? undefined; + } else if (ref.id.match(tmdbRegex)) { + const tmdbMatch = ref.id.match(tmdbRegex)?.[1]; + mediaIds.tmdbId = Number(tmdbMatch); + } else if (ref.id.match(tvdbRegex)) { + const tvdbMatch = ref.id.match(tvdbRegex)?.[1]; + mediaIds.tvdbId = Number(tvdbMatch); + } + }); + + // If we got an IMDb ID, but no TMDb ID, lookup the TMDb ID with the IMDb ID + if (mediaIds.imdbId && !mediaIds.tmdbId) { + const tmdbMovie = await this.tmdb.getMovieByImdbId({ + imdbId: mediaIds.imdbId, + }); + mediaIds.tmdbId = tmdbMovie.id; + } + // Check if the agent is IMDb + } else if (plexitem.guid.match(imdbRegex)) { + const imdbMatch = plexitem.guid.match(imdbRegex); + if (imdbMatch) { + mediaIds.imdbId = imdbMatch[1]; + const tmdbMovie = await this.tmdb.getMovieByImdbId({ + imdbId: mediaIds.imdbId, + }); + mediaIds.tmdbId = tmdbMovie.id; + } + // Check if the agent is TMDb + } else if (plexitem.guid.match(tmdbRegex)) { + const tmdbMatch = plexitem.guid.match(tmdbRegex); + if (tmdbMatch) { + mediaIds.tmdbId = Number(tmdbMatch[1]); + } + // Check if the agent is TVDb + } else if (plexitem.guid.match(tvdbRegex)) { + const matchedtvdb = plexitem.guid.match(tvdbRegex); + + // If we can find a tvdb Id, use it to get the full tmdb show details + if (matchedtvdb) { + const show = await this.tmdb.getShowByTvdbId({ + tvdbId: Number(matchedtvdb[1]), + }); + + mediaIds.tvdbId = Number(matchedtvdb[1]); + mediaIds.tmdbId = show.id; + } + // Check if the agent (for shows) is TMDb + } else if (plexitem.guid.match(tmdbShowRegex)) { + const matchedtmdb = plexitem.guid.match(tmdbShowRegex); + if (matchedtmdb) { + mediaIds.tmdbId = Number(matchedtmdb[1]); + } + // Check for HAMA (with TVDb guid) + } else if (plexitem.guid.match(hamaTvdbRegex)) { + const matchedtvdb = plexitem.guid.match(hamaTvdbRegex); + + if (matchedtvdb) { + const show = await this.tmdb.getShowByTvdbId({ + tvdbId: Number(matchedtvdb[1]), + }); + + mediaIds.tvdbId = Number(matchedtvdb[1]); + mediaIds.tmdbId = show.id; + // Set isHama to true, so we can know to add special processing to this item + mediaIds.isHama = true; + } + // Check for HAMA (with anidb guid) + } else if (plexitem.guid.match(hamaAnidbRegex)) { + const matchedhama = plexitem.guid.match(hamaAnidbRegex); + + if (!animeList.isLoaded()) { + this.log( + `Hama ID ${plexitem.guid} detected, but library agent is not set to Hama`, + 'warn', + { title: plexitem.title } + ); + } else if (matchedhama) { + const anidbId = Number(matchedhama[1]); + const result = animeList.getFromAnidbId(anidbId); + let tvShow: TmdbTvDetails | null = null; + + // Set isHama to true, so we can know to add special processing to this item + mediaIds.isHama = true; + + // First try to lookup the show by TVDb ID + if (result?.tvdbId) { + const extResponse = await this.tmdb.getByExternalId({ + externalId: result.tvdbId, + type: 'tvdb', + }); + if (extResponse.tv_results[0]) { + tvShow = await this.tmdb.getTvShow({ + tvId: extResponse.tv_results[0].id, + }); + mediaIds.tvdbId = result.tvdbId; + mediaIds.tmdbId = tvShow.id; + } else { + this.log( + `Missing TVDB ${result.tvdbId} entry in TMDB for AniDB ${anidbId}` + ); + } + } + + if (!tvShow) { + // if lookup of tvshow above failed, then try movie with tmdbid/imdbid + // note - some tv shows have imdbid set too, that's why this need to go second + if (result?.tmdbId) { + mediaIds.tmdbId = result.tmdbId; + mediaIds.imdbId = result?.imdbId; + } else if (result?.imdbId) { + const tmdbMovie = await this.tmdb.getMovieByImdbId({ + imdbId: result.imdbId, + }); + mediaIds.tmdbId = tmdbMovie.id; + mediaIds.imdbId = result.imdbId; + } + } + } + } + + if (!mediaIds.tmdbId) { + throw new Error('Unable to find TMDb ID'); + } + + // We check above if we have the TMDb ID, so we can safely assert the type below + return mediaIds as MediaIds; + } + + // movies with hama agent actually are tv shows with at least one episode in it + // try to get first episode of any season - cannot hardcode season or episode number + // because sometimes user can have it in other season/ep than s01e01 + private async processHamaMovie(metadata: PlexMetadata, tmdbId: number) { + const season = metadata.Children?.Metadata[0]; + if (season) { + const episodes = await this.plexClient.getChildrenMetadata( + season.ratingKey + ); + if (episodes) { + await this.processPlexMovieByTmdbId(episodes[0], tmdbId); + } + } + } + + // this adds all movie episodes from specials season for Hama agent + private async processHamaSpecials(metadata: PlexMetadata, tvdbId: number) { + const specials = metadata.Children?.Metadata.find( + (md) => Number(md.index) === 0 + ); + if (specials) { + const episodes = await this.plexClient.getChildrenMetadata( + specials.ratingKey + ); + if (episodes) { + for (const episode of episodes) { + const special = animeList.getSpecialEpisode(tvdbId, episode.index); + if (special) { + if (special.tmdbId) { + await this.processPlexMovieByTmdbId(episode, special.tmdbId); + } else if (special.imdbId) { + const tmdbMovie = await this.tmdb.getMovieByImdbId({ + imdbId: special.imdbId, + }); + await this.processPlexMovieByTmdbId(episode, tmdbMovie.id); + } + } + } + } + } + } + + // checks if any of this.libraries has Hama agent set in Plex + private async hasHamaAgent() { + const plexLibraries = await this.plexClient.getLibraries(); + return this.libraries.some((library) => + plexLibraries.some( + (plexLibrary) => + plexLibrary.agent === HAMA_AGENT && library.id === plexLibrary.key + ) + ); + } +} + +export const plexFullScanner = new PlexScanner(); +export const plexRecentScanner = new PlexScanner(true); diff --git a/server/lib/scanners/radarr/index.ts b/server/lib/scanners/radarr/index.ts new file mode 100644 index 00000000..74682cc5 --- /dev/null +++ b/server/lib/scanners/radarr/index.ts @@ -0,0 +1,94 @@ +import { uniqWith } from 'lodash'; +import RadarrAPI, { RadarrMovie } from '../../../api/radarr'; +import { getSettings, RadarrSettings } from '../../settings'; +import BaseScanner, { RunnableScanner, StatusBase } from '../baseScanner'; + +type SyncStatus = StatusBase & { + currentServer: RadarrSettings; + servers: RadarrSettings[]; +}; + +class RadarrScanner + extends BaseScanner + implements RunnableScanner { + private servers: RadarrSettings[]; + private currentServer: RadarrSettings; + private radarrApi: RadarrAPI; + + constructor() { + super('Radarr Scan', { bundleSize: 50 }); + } + + public status(): SyncStatus { + return { + running: this.running, + progress: this.progress, + total: this.items.length, + currentServer: this.currentServer, + servers: this.servers, + }; + } + + public async run(): Promise { + const settings = getSettings(); + const sessionId = this.startRun(); + + try { + this.servers = uniqWith(settings.radarr, (radarrA, radarrB) => { + return ( + radarrA.hostname === radarrB.hostname && + radarrA.port === radarrB.port && + radarrA.baseUrl === radarrB.baseUrl + ); + }); + + for (const server of this.servers) { + this.currentServer = server; + if (server.syncEnabled) { + this.log( + `Beginning to process Radarr server: ${server.name}`, + 'info' + ); + + this.radarrApi = new RadarrAPI({ + apiKey: server.apiKey, + url: RadarrAPI.buildRadarrUrl(server, '/api/v3'), + }); + + this.items = await this.radarrApi.getMovies(); + + await this.loop(this.processRadarrMovie.bind(this), { sessionId }); + } else { + this.log(`Sync not enabled. Skipping Radarr server: ${server.name}`); + } + } + + this.log('Radarr scan complete', 'info'); + } catch (e) { + this.log('Scan interrupted', 'error', { errorMessage: e.message }); + } finally { + this.endRun(sessionId); + } + } + + private async processRadarrMovie(radarrMovie: RadarrMovie): Promise { + try { + const server4k = this.enable4kMovie && this.currentServer.is4k; + await this.processMovie(radarrMovie.tmdbId, { + is4k: server4k, + serviceId: this.currentServer.id, + externalServiceId: radarrMovie.id, + externalServiceSlug: radarrMovie.titleSlug, + title: radarrMovie.title, + processing: !radarrMovie.downloaded, + }); + } catch (e) { + this.log('Failed to process Radarr media', 'error', { + errorMessage: e.message, + title: radarrMovie.title, + }); + } + } +} + +export const radarrScanner = new RadarrScanner(); diff --git a/server/lib/scanners/sonarr/index.ts b/server/lib/scanners/sonarr/index.ts new file mode 100644 index 00000000..4bc505fb --- /dev/null +++ b/server/lib/scanners/sonarr/index.ts @@ -0,0 +1,134 @@ +import { uniqWith } from 'lodash'; +import { getRepository } from 'typeorm'; +import SonarrAPI, { SonarrSeries } from '../../../api/sonarr'; +import Media from '../../../entity/Media'; +import { getSettings, SonarrSettings } from '../../settings'; +import BaseScanner, { + ProcessableSeason, + RunnableScanner, + StatusBase, +} from '../baseScanner'; + +type SyncStatus = StatusBase & { + currentServer: SonarrSettings; + servers: SonarrSettings[]; +}; + +class SonarrScanner + extends BaseScanner + implements RunnableScanner { + private servers: SonarrSettings[]; + private currentServer: SonarrSettings; + private sonarrApi: SonarrAPI; + + constructor() { + super('Sonarr Scan', { bundleSize: 50 }); + } + + public status(): SyncStatus { + return { + running: this.running, + progress: this.progress, + total: this.items.length, + currentServer: this.currentServer, + servers: this.servers, + }; + } + + public async run(): Promise { + const settings = getSettings(); + const sessionId = this.startRun(); + + try { + this.servers = uniqWith(settings.sonarr, (sonarrA, sonarrB) => { + return ( + sonarrA.hostname === sonarrB.hostname && + sonarrA.port === sonarrB.port && + sonarrA.baseUrl === sonarrB.baseUrl + ); + }); + + for (const server of this.servers) { + this.currentServer = server; + if (server.syncEnabled) { + this.log( + `Beginning to process Sonarr server: ${server.name}`, + 'info' + ); + + this.sonarrApi = new SonarrAPI({ + apiKey: server.apiKey, + url: SonarrAPI.buildSonarrUrl(server, '/api/v3'), + }); + + this.items = await this.sonarrApi.getSeries(); + + await this.loop(this.processSonarrSeries.bind(this), { sessionId }); + } else { + this.log(`Sync not enabled. Skipping Sonarr server: ${server.name}`); + } + } + + this.log('Sonarr scan complete', 'info'); + } catch (e) { + this.log('Scan interrupted', 'error', { errorMessage: e.message }); + } finally { + this.endRun(sessionId); + } + } + + private async processSonarrSeries(sonarrSeries: SonarrSeries) { + try { + const mediaRepository = getRepository(Media); + const server4k = this.enable4kShow && this.currentServer.is4k; + const processableSeasons: ProcessableSeason[] = []; + let tmdbId: number; + + const media = await mediaRepository.findOne({ + where: { tvdbId: sonarrSeries.tvdbId }, + }); + + if (!media || !media.tmdbId) { + const tvShow = await this.tmdb.getShowByTvdbId({ + tvdbId: sonarrSeries.tvdbId, + }); + + tmdbId = tvShow.id; + } else { + tmdbId = media.tmdbId; + } + + const filteredSeasons = sonarrSeries.seasons.filter( + (sn) => sn.seasonNumber !== 0 + ); + + for (const season of filteredSeasons) { + const totalAvailableEpisodes = season.statistics?.episodeFileCount ?? 0; + + processableSeasons.push({ + seasonNumber: season.seasonNumber, + episodes: !server4k ? totalAvailableEpisodes : 0, + episodes4k: server4k ? totalAvailableEpisodes : 0, + totalEpisodes: season.statistics?.totalEpisodeCount ?? 0, + processing: season.monitored && totalAvailableEpisodes === 0, + is4kOverride: server4k, + }); + } + + await this.processShow(tmdbId, sonarrSeries.tvdbId, processableSeasons, { + serviceId: this.currentServer.id, + externalServiceId: sonarrSeries.id, + externalServiceSlug: sonarrSeries.titleSlug, + title: sonarrSeries.title, + is4k: server4k, + }); + } catch (e) { + this.log('Failed to process Sonarr media', 'error', { + errorMessage: e.message, + title: sonarrSeries.title, + }); + } + } +} + +export const sonarrScanner = new SonarrScanner(); diff --git a/server/routes/settings/index.ts b/server/routes/settings/index.ts index 0099d28c..a7dbd3c1 100644 --- a/server/routes/settings/index.ts +++ b/server/routes/settings/index.ts @@ -4,7 +4,6 @@ import { getRepository } from 'typeorm'; import { User } from '../../entity/User'; import PlexAPI from '../../api/plexapi'; import PlexTvAPI from '../../api/plextv'; -import { jobPlexFullSync } from '../../job/plexsync'; import { scheduledJobs } from '../../job/schedule'; import { Permission } from '../../lib/permissions'; import { isAuthenticated } from '../../middleware/auth'; @@ -17,6 +16,7 @@ import notificationRoutes from './notifications'; import sonarrRoutes from './sonarr'; import radarrRoutes from './radarr'; import cacheManager, { AvailableCacheIds } from '../../lib/cache'; +import { plexFullScanner } from '../../lib/scanners/plex'; const settingsRoutes = Router(); @@ -211,16 +211,16 @@ settingsRoutes.get('/plex/library', async (req, res) => { }); settingsRoutes.get('/plex/sync', (_req, res) => { - return res.status(200).json(jobPlexFullSync.status()); + return res.status(200).json(plexFullScanner.status()); }); settingsRoutes.post('/plex/sync', (req, res) => { if (req.body.cancel) { - jobPlexFullSync.cancel(); + plexFullScanner.cancel(); } else if (req.body.start) { - jobPlexFullSync.run(); + plexFullScanner.run(); } - return res.status(200).json(jobPlexFullSync.status()); + return res.status(200).json(plexFullScanner.status()); }); settingsRoutes.get('/jobs', (_req, res) => {