feat: map AniDB IDs from Hama agent to tvdb/tmdb/imdb IDs (#538)
* feat: map AniDB IDs from Hama agent to tvdb/tmdb/imdb IDs re #453 * refactor: removes sync job for AnimeList, load mapping on demand * refactor: addressing review comments, using typescript types for xml parsing * refactor: make sure sync job does not update create same tvshow/movie twice Hama agent can have same tvdbid it for different library items - for example when user stores different seasons for same tv show separately. This change adds "AsyncLock" that guarantees code in callback runs for same id fully, before running same callback next time. * refactor: do not use season 0 tvdbid for tvshow from mapping file * refactor: support multiple imdb mappings for same anidb entry * refactor: add debug log for missing tvdb entries in tmdb lookups from anidb/hama agentpull/596/head
Mārtiņš Možeiko
3 years ago
committed by
GitHub
parent
2bfe0f2bf6
commit
0600ac7c3a
@ -0,0 +1,223 @@
|
||||
import axios from 'axios';
|
||||
import xml2js from 'xml2js';
|
||||
import fs, { promises as fsp } from 'fs';
|
||||
import path from 'path';
|
||||
import logger from '../logger';
|
||||
|
||||
const UPDATE_INTERVAL_MSEC = 24 * 3600 * 1000; // how often to download new mapping in milliseconds
|
||||
// originally at https://raw.githubusercontent.com/ScudLee/anime-lists/master/anime-list.xml
|
||||
const MAPPING_URL =
|
||||
'https://raw.githubusercontent.com/Anime-Lists/anime-lists/master/anime-list.xml';
|
||||
const LOCAL_PATH = path.join(__dirname, '../../config/anime-list.xml');
|
||||
|
||||
const mappingRegexp = new RegExp(/;[0-9]+-([0-9]+)/g);
|
||||
|
||||
// Anime-List xml files are community maintained mappings that Hama agent uses to map AniDB IDs to tvdb/tmdb IDs
|
||||
// https://github.com/Anime-Lists/anime-lists/
|
||||
|
||||
interface AnimeMapping {
|
||||
$: {
|
||||
anidbseason: string;
|
||||
tvdbseason: string;
|
||||
};
|
||||
_: string;
|
||||
}
|
||||
|
||||
interface Anime {
|
||||
$: {
|
||||
anidbid: number;
|
||||
tvdbid?: string;
|
||||
defaulttvdbseason?: string;
|
||||
tmdbid?: number;
|
||||
imdbid?: string;
|
||||
};
|
||||
'mapping-list'?: {
|
||||
mapping: AnimeMapping[];
|
||||
}[];
|
||||
}
|
||||
|
||||
interface AnimeList {
|
||||
'anime-list': {
|
||||
anime: Anime[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface AnidbItem {
|
||||
tvdbId?: number;
|
||||
tmdbId?: number;
|
||||
imdbId?: string;
|
||||
}
|
||||
|
||||
class AnimeListMapping {
|
||||
private syncing = false;
|
||||
|
||||
private mapping: { [anidbId: number]: AnidbItem } = {};
|
||||
|
||||
// mapping file modification date when it was loaded
|
||||
private mappingModified: Date | null = null;
|
||||
|
||||
// each episode in season 0 from TVDB can map to movie
|
||||
private specials: { [tvdbId: number]: { [episode: number]: AnidbItem } } = {};
|
||||
|
||||
public isLoaded = () => Object.keys(this.mapping).length !== 0;
|
||||
|
||||
private loadFromFile = async () => {
|
||||
logger.info('Loading mapping file', { label: 'Anime-List Sync' });
|
||||
try {
|
||||
const mappingStat = await fsp.stat(LOCAL_PATH);
|
||||
const file = await fsp.readFile(LOCAL_PATH);
|
||||
const xml = (await xml2js.parseStringPromise(file)) as AnimeList;
|
||||
|
||||
this.mapping = {};
|
||||
this.specials = {};
|
||||
for (const anime of xml['anime-list'].anime) {
|
||||
// tvdbId can be nonnumber, like 'movie' string
|
||||
let tvdbId: number | undefined;
|
||||
if (anime.$.tvdbid && !isNaN(Number(anime.$.tvdbid))) {
|
||||
tvdbId = Number(anime.$.tvdbid);
|
||||
} else {
|
||||
tvdbId = undefined;
|
||||
}
|
||||
|
||||
let imdbIds: (string | undefined)[];
|
||||
if (anime.$.imdbid) {
|
||||
// if there are multiple imdb entries, then they map to different movies
|
||||
imdbIds = anime.$.imdbid.split(',');
|
||||
} else {
|
||||
// in case there is no imdbid, that's ok as there will be tmdbid
|
||||
imdbIds = [undefined];
|
||||
}
|
||||
|
||||
const tmdbId = anime.$.tmdbid ? Number(anime.$.tmdbid) : undefined;
|
||||
const anidbId = Number(anime.$.anidbid);
|
||||
this.mapping[anidbId] = {
|
||||
// for season 0 ignore tvdbid, because this must be movie/OVA
|
||||
tvdbId: anime.$.defaulttvdbseason === '0' ? undefined : tvdbId,
|
||||
tmdbId: tmdbId,
|
||||
imdbId: imdbIds[0], // this is used for one AniDB -> one imdb movie mapping
|
||||
};
|
||||
|
||||
if (tvdbId) {
|
||||
const mappingList = anime['mapping-list'];
|
||||
if (mappingList && mappingList.length != 0) {
|
||||
let imdbIndex = 0;
|
||||
for (const mapping of mappingList[0].mapping) {
|
||||
const text = mapping._;
|
||||
if (text && mapping.$.tvdbseason === '0') {
|
||||
let matches;
|
||||
while ((matches = mappingRegexp.exec(text)) !== null) {
|
||||
const episode = Number(matches[1]);
|
||||
if (!this.specials[tvdbId]) {
|
||||
this.specials[tvdbId] = {};
|
||||
}
|
||||
// map next available imdbid to episode in s0
|
||||
const imdbId =
|
||||
imdbIndex > imdbIds.length ? undefined : imdbIds[imdbIndex];
|
||||
if (tmdbId || imdbId) {
|
||||
this.specials[tvdbId][episode] = {
|
||||
tmdbId: tmdbId,
|
||||
imdbId: imdbId,
|
||||
};
|
||||
imdbIndex++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// some movies do not have mapping-list, so map episode 1,2,3,..to movies
|
||||
// movies must have imdbid or tmdbid
|
||||
const hasImdb = imdbIds.length > 1 || imdbIds[0] !== undefined;
|
||||
if ((hasImdb || tmdbId) && anime.$.defaulttvdbseason === '0') {
|
||||
if (!this.specials[tvdbId]) {
|
||||
this.specials[tvdbId] = {};
|
||||
}
|
||||
// map each imdbid to episode in s0, episode index starts with 1
|
||||
for (let idx = 0; idx < imdbIds.length; idx++) {
|
||||
this.specials[tvdbId][idx + 1] = {
|
||||
tmdbId: tmdbId,
|
||||
imdbId: imdbIds[idx],
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
this.mappingModified = mappingStat.mtime;
|
||||
logger.info(
|
||||
`Loaded ${
|
||||
Object.keys(this.mapping).length
|
||||
} AniDB items from mapping file`,
|
||||
{ label: 'Anime-List Sync' }
|
||||
);
|
||||
} catch (e) {
|
||||
throw new Error(`Failed to load Anime-List mappings: ${e.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
private downloadFile = async () => {
|
||||
logger.info('Downloading latest mapping file', {
|
||||
label: 'Anime-List Sync',
|
||||
});
|
||||
try {
|
||||
const response = await axios.get(MAPPING_URL, {
|
||||
responseType: 'stream',
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
const writer = fs.createWriteStream(LOCAL_PATH);
|
||||
writer.on('finish', resolve);
|
||||
response.data.pipe(writer);
|
||||
});
|
||||
} catch (e) {
|
||||
throw new Error(`Failed to download Anime-List mapping: ${e.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
public sync = async () => {
|
||||
// make sure only one sync runs at a time
|
||||
if (this.syncing) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.syncing = true;
|
||||
try {
|
||||
// check if local file is not "expired" yet
|
||||
if (fs.existsSync(LOCAL_PATH)) {
|
||||
const now = new Date();
|
||||
const stat = await fsp.stat(LOCAL_PATH);
|
||||
if (now.getTime() - stat.mtime.getTime() < UPDATE_INTERVAL_MSEC) {
|
||||
if (!this.isLoaded()) {
|
||||
// no need to download, but make sure file is loaded
|
||||
await this.loadFromFile();
|
||||
} else if (
|
||||
this.mappingModified &&
|
||||
stat.mtime.getTime() > this.mappingModified.getTime()
|
||||
) {
|
||||
// if file has been modified externally since last load, reload it
|
||||
await this.loadFromFile();
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
await this.downloadFile();
|
||||
await this.loadFromFile();
|
||||
} finally {
|
||||
this.syncing = false;
|
||||
}
|
||||
};
|
||||
|
||||
public getFromAnidbId = (anidbId: number): AnidbItem | undefined => {
|
||||
return this.mapping[anidbId];
|
||||
};
|
||||
|
||||
public getSpecialEpisode = (
|
||||
tvdbId: number,
|
||||
episode: number
|
||||
): AnidbItem | undefined => {
|
||||
const episodes = this.specials[tvdbId];
|
||||
return episodes ? episodes[episode] : undefined;
|
||||
};
|
||||
}
|
||||
|
||||
const animeList = new AnimeListMapping();
|
||||
|
||||
export default animeList;
|
||||
@ -0,0 +1,54 @@
|
||||
import { EventEmitter } from 'events';
|
||||
|
||||
// whenever you need to run async code on tv show or movie that does "get existing" / "check if need to create new" / "save"
|
||||
// then you need to put all of that code in "await asyncLock.dispatch" callback based on media id
|
||||
// this will guarantee that only one part of code will run at the same for this media id to avoid code
|
||||
// trying to create two or more entries for same movie/tvshow (which would result in sqlite unique constraint failrue)
|
||||
|
||||
class AsyncLock {
|
||||
private locked: { [key: string]: boolean } = {};
|
||||
private ee = new EventEmitter();
|
||||
|
||||
constructor() {
|
||||
this.ee.setMaxListeners(0);
|
||||
}
|
||||
|
||||
private acquire = async (key: string) => {
|
||||
return new Promise((resolve) => {
|
||||
if (!this.locked[key]) {
|
||||
this.locked[key] = true;
|
||||
return resolve(undefined);
|
||||
}
|
||||
|
||||
const nextAcquire = () => {
|
||||
if (!this.locked[key]) {
|
||||
this.locked[key] = true;
|
||||
this.ee.removeListener(key, nextAcquire);
|
||||
return resolve(undefined);
|
||||
}
|
||||
};
|
||||
|
||||
this.ee.on(key, nextAcquire);
|
||||
});
|
||||
};
|
||||
|
||||
private release = (key: string): void => {
|
||||
delete this.locked[key];
|
||||
setImmediate(() => this.ee.emit(key));
|
||||
};
|
||||
|
||||
public dispatch = async (
|
||||
key: string | number,
|
||||
callback: () => Promise<void>
|
||||
) => {
|
||||
const skey = String(key);
|
||||
await this.acquire(skey);
|
||||
try {
|
||||
await callback();
|
||||
} finally {
|
||||
this.release(skey);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export default AsyncLock;
|
||||
Loading…
Reference in new issue