Improved the ffprobe call caching mechanism by storing result to DB and using it for indexing and subtitles search.
morpheus65535
4 years ago
parent
887da10d9b
commit
33e1555311
@ -1,68 +1,111 @@
|
|||||||
# coding=utf-8
|
# coding=utf-8
|
||||||
|
|
||||||
import enzyme
|
|
||||||
from enzyme.exceptions import MalformedMKVError
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import datetime
|
import pickle
|
||||||
from knowit import api
|
from knowit import api
|
||||||
from subliminal.cache import region
|
import enzyme
|
||||||
|
from enzyme.exceptions import MalformedMKVError
|
||||||
|
from enzyme.exceptions import MalformedMKVError
|
||||||
|
from database import database
|
||||||
|
|
||||||
|
|
||||||
|
def embedded_subs_reader(file, file_size, episode_file_id=None, movie_file_id=None):
|
||||||
|
data = parse_video_metadata(file, file_size, episode_file_id, movie_file_id)
|
||||||
|
|
||||||
FFPROBE_CACHE_EXPIRATION_TIME = datetime.timedelta(weeks=2).total_seconds()
|
subtitles_list = []
|
||||||
|
if data['ffprobe']:
|
||||||
|
traditional_chinese = ["cht", "tc", "traditional", "zht", "hant", "big5", u"繁", u"雙語"]
|
||||||
|
brazilian_portuguese = ["pt-br", "pob", "pb", "brazilian", "brasil", "brazil"]
|
||||||
|
|
||||||
|
if 'subtitle' in data['ffprobe']:
|
||||||
|
for detected_language in data['ffprobe']['subtitle']:
|
||||||
|
if 'language' in detected_language:
|
||||||
|
language = detected_language['language'].alpha3
|
||||||
|
if language == 'zho' and 'name' in detected_language:
|
||||||
|
if any (ext in (detected_language['name'].lower()) for ext in traditional_chinese):
|
||||||
|
language = 'zht'
|
||||||
|
if language == 'por' and 'name' in detected_language:
|
||||||
|
if any (ext in (detected_language['name'].lower()) for ext in brazilian_portuguese):
|
||||||
|
language = 'pob'
|
||||||
|
forced = detected_language['forced'] if 'forced' in detected_language else False
|
||||||
|
hearing_impaired = detected_language['hearing_impaired'] if 'hearing_impaired' in \
|
||||||
|
detected_language else False
|
||||||
|
codec = detected_language['format'] if 'format' in detected_language else None
|
||||||
|
subtitles_list.append([language, forced, hearing_impaired, codec])
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
elif data['enzyme']:
|
||||||
|
for subtitle_track in data['enzyme'].subtitle_tracks:
|
||||||
|
hearing_impaired = False
|
||||||
|
if subtitle_track.name:
|
||||||
|
if 'sdh' in subtitle_track.name.lower():
|
||||||
|
hearing_impaired = True
|
||||||
|
subtitles_list.append([subtitle_track.language, subtitle_track.forced, hearing_impaired,
|
||||||
|
subtitle_track.codec_id])
|
||||||
|
|
||||||
class EmbeddedSubsReader:
|
return subtitles_list
|
||||||
def __init__(self):
|
|
||||||
self.ffprobe = None
|
|
||||||
|
|
||||||
@region.cache_on_arguments(expiration_time=FFPROBE_CACHE_EXPIRATION_TIME)
|
|
||||||
# file_size, episode_file_id and movie_file_id are used for cache identification. DO NOT REMOVE!
|
|
||||||
def list_languages(self, file, file_size, episode_file_id=None, movie_file_id=None):
|
|
||||||
from utils import get_binary
|
|
||||||
self.ffprobe = get_binary("ffprobe")
|
|
||||||
|
|
||||||
subtitles_list = []
|
def parse_video_metadata(file, file_size, episode_file_id=None, movie_file_id=None):
|
||||||
if self.ffprobe:
|
# Define default data keys value
|
||||||
api.initialize({'provider': 'ffmpeg', 'ffmpeg': self.ffprobe})
|
data = {
|
||||||
data = api.know(file)
|
'ffprobe': {},
|
||||||
|
'enzyme': {},
|
||||||
|
'file_id': episode_file_id if episode_file_id else movie_file_id,
|
||||||
|
'file_size': file_size
|
||||||
|
}
|
||||||
|
|
||||||
traditional_chinese = ["cht", "tc", "traditional", "zht", "hant", "big5", u"繁", u"雙語"]
|
# Get the actual cache value form database
|
||||||
brazilian_portuguese = ["pt-br", "pob", "pb", "brazilian", "brasil", "brazil"]
|
if episode_file_id:
|
||||||
|
cache_key = database.execute('SELECT ffprobe_cache FROM table_episodes WHERE episode_file_id=? AND file_size=?',
|
||||||
|
(episode_file_id, file_size), only_one=True)
|
||||||
|
elif movie_file_id:
|
||||||
|
cache_key = database.execute('SELECT ffprobe_cache FROM table_movies WHERE movie_file_id=? AND file_size=?',
|
||||||
|
(movie_file_id, file_size), only_one=True)
|
||||||
|
else:
|
||||||
|
cache_key = None
|
||||||
|
|
||||||
if 'subtitle' in data:
|
# check if we have a value for that cache key
|
||||||
for detected_language in data['subtitle']:
|
if not isinstance(cache_key, dict):
|
||||||
if 'language' in detected_language:
|
return data
|
||||||
language = detected_language['language'].alpha3
|
else:
|
||||||
if language == 'zho' and 'name' in detected_language:
|
try:
|
||||||
if any (ext in (detected_language['name'].lower()) for ext in traditional_chinese):
|
# Unpickle ffprobe cache
|
||||||
language = 'zht'
|
cached_value = pickle.loads(cache_key['ffprobe_cache'])
|
||||||
if language == 'por' and 'name' in detected_language:
|
except:
|
||||||
if any (ext in (detected_language['name'].lower()) for ext in brazilian_portuguese):
|
pass
|
||||||
language = 'pob'
|
|
||||||
forced = detected_language['forced'] if 'forced' in detected_language else False
|
|
||||||
hearing_impaired = detected_language['hearing_impaired'] if 'hearing_impaired' in \
|
|
||||||
detected_language else False
|
|
||||||
codec = detected_language['format'] if 'format' in detected_language else None
|
|
||||||
subtitles_list.append([language, forced, hearing_impaired, codec])
|
|
||||||
else:
|
|
||||||
continue
|
|
||||||
else:
|
else:
|
||||||
if os.path.splitext(file)[1] == '.mkv':
|
# Check if file size and file id matches and if so, we return the cached value
|
||||||
with open(file, 'rb') as f:
|
if cached_value['file_size'] == file_size and cached_value['file_id'] in [episode_file_id, movie_file_id]:
|
||||||
try:
|
return cached_value
|
||||||
mkv = enzyme.MKV(f)
|
|
||||||
except MalformedMKVError:
|
|
||||||
logging.error('BAZARR cannot analyze this MKV with our built-in MKV parser, you should install ffmpeg: ' + file)
|
|
||||||
else:
|
|
||||||
for subtitle_track in mkv.subtitle_tracks:
|
|
||||||
hearing_impaired = False
|
|
||||||
if subtitle_track.name:
|
|
||||||
if 'sdh' in subtitle_track.name.lower():
|
|
||||||
hearing_impaired = True
|
|
||||||
subtitles_list.append([subtitle_track.language, subtitle_track.forced, hearing_impaired,
|
|
||||||
subtitle_track.codec_id])
|
|
||||||
|
|
||||||
return subtitles_list
|
# if not, we retrieve the metadata from the file
|
||||||
|
from utils import get_binary
|
||||||
|
ffprobe_path = get_binary("ffprobe")
|
||||||
|
|
||||||
|
# if we have ffprobe available
|
||||||
|
if ffprobe_path:
|
||||||
|
api.initialize({'provider': 'ffmpeg', 'ffmpeg': ffprobe_path})
|
||||||
|
data['ffprobe'] = api.know(file)
|
||||||
|
# if nto, we use enzyme for mkv files
|
||||||
|
else:
|
||||||
|
if os.path.splitext(file)[1] == '.mkv':
|
||||||
|
with open(file, 'rb') as f:
|
||||||
|
try:
|
||||||
|
mkv = enzyme.MKV(f)
|
||||||
|
except MalformedMKVError:
|
||||||
|
logging.error(
|
||||||
|
'BAZARR cannot analyze this MKV with our built-in MKV parser, you should install '
|
||||||
|
'ffmpeg/ffprobe: ' + file)
|
||||||
|
else:
|
||||||
|
data['enzyme'] = mkv
|
||||||
|
|
||||||
embedded_subs_reader = EmbeddedSubsReader()
|
# we write to db the result and return the newly cached ffprobe dict
|
||||||
|
if episode_file_id:
|
||||||
|
database.execute('UPDATE table_episodes SET ffprobe_cache=? WHERE episode_file_id=?',
|
||||||
|
(pickle.dumps(data, pickle.HIGHEST_PROTOCOL), episode_file_id))
|
||||||
|
elif movie_file_id:
|
||||||
|
database.execute('UPDATE table_movies SET ffprobe_cache=? WHERE movie_file_id=?',
|
||||||
|
(pickle.dumps(data, pickle.HIGHEST_PROTOCOL), movie_file_id))
|
||||||
|
return data
|
||||||
|
|||||||
Loading…
Reference in new issue