From 9dd85eeee703e7663ac45b37878bcdfe1e9d0cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Sun, 22 Sep 2019 20:35:12 -0400 Subject: [PATCH] WIP --- libs/fcache/cache.py | 72 ++++++++++++++----- libs/subliminal_patch/providers/legendastv.py | 4 +- views/wanted.tpl | 1 + 3 files changed, 57 insertions(+), 20 deletions(-) diff --git a/libs/fcache/cache.py b/libs/fcache/cache.py index e1510c233..695f916c3 100644 --- a/libs/fcache/cache.py +++ b/libs/fcache/cache.py @@ -4,9 +4,13 @@ import os import pickle import shutil import tempfile +import traceback +import hashlib import appdirs +from scandir import scandir, scandir_generic as _scandir_generic + try: from collections.abc import MutableMapping unicode = str @@ -86,7 +90,7 @@ class FileCache(MutableMapping): """ def __init__(self, appname, flag='c', mode=0o666, keyencoding='utf-8', - serialize=True, app_cache_dir=None): + serialize=True, app_cache_dir=None, key_file_ext=".txt"): """Initialize a :class:`FileCache` object.""" if not isinstance(flag, str): raise TypeError("flag must be str not '{}'".format(type(flag))) @@ -127,6 +131,7 @@ class FileCache(MutableMapping): self._mode = mode self._keyencoding = keyencoding self._serialize = serialize + self.key_file_ext = key_file_ext def _parse_appname(self, appname): """Splits an appname into the appname and subcache components.""" @@ -180,7 +185,16 @@ class FileCache(MutableMapping): self._sync = True for ekey in self._buffer: filename = self._key_to_filename(ekey) - self._write_to_file(filename, self._buffer[ekey]) + try: + self._write_to_file(filename, self._buffer[ekey]) + except: + logger.error("Couldn't write content from %r to cache file: %r: %s", ekey, filename, + traceback.format_exc()) + try: + self.__write_to_file(filename + self.key_file_ext, ekey) + except: + logger.error("Couldn't write content from %r to cache file: %r: %s", ekey, filename, + traceback.format_exc()) self._buffer.clear() self._sync = False @@ -189,8 +203,7 @@ class FileCache(MutableMapping): raise ValueError("invalid operation on closed cache") def _encode_key(self, key): - """Encode key using *hex_codec* for constructing a cache filename. - + """ Keys are implicitly converted to :class:`bytes` if passed as :class:`str`. @@ -199,16 +212,15 @@ class FileCache(MutableMapping): key = key.encode(self._keyencoding) elif not isinstance(key, bytes): raise TypeError("key must be bytes or str") - return codecs.encode(key, 'hex_codec').decode(self._keyencoding) + return key.decode(self._keyencoding) def _decode_key(self, key): - """Decode key using hex_codec to retrieve the original key. - + """ Keys are returned as :class:`str` if serialization is enabled. Keys are returned as :class:`bytes` if serialization is disabled. """ - bkey = codecs.decode(key.encode(self._keyencoding), 'hex_codec') + bkey = key.encode(self._keyencoding) return bkey.decode(self._keyencoding) if self._serialize else bkey def _dumps(self, value): @@ -219,19 +231,27 @@ class FileCache(MutableMapping): def _key_to_filename(self, key): """Convert an encoded key to an absolute cache filename.""" - return os.path.join(self.cache_dir, key) + if isinstance(key, unicode): + key = key.encode(self._keyencoding) + return os.path.join(self.cache_dir, hashlib.md5(key).hexdigest()) def _filename_to_key(self, absfilename): """Convert an absolute cache filename to a key name.""" - return os.path.split(absfilename)[1] + hkey_hdr_fn = absfilename + self.key_file_ext + if os.path.isfile(hkey_hdr_fn): + with open(hkey_hdr_fn, 'rb') as f: + key = f.read() + return key.decode(self._keyencoding) if self._serialize else key - def _all_filenames(self): + def _all_filenames(self, scandir_generic=True): """Return a list of absolute cache filenames""" + _scandir = _scandir_generic if scandir_generic else scandir try: - return [os.path.join(self.cache_dir, filename) for filename in - os.listdir(self.cache_dir)] + for entry in _scandir(self.cache_dir): + if entry.is_file(follow_symlinks=False) and not entry.name.endswith(self.key_file_ext): + yield os.path.join(self.cache_dir, entry.name) except (FileNotFoundError, OSError): - return [] + raise StopIteration def _all_keys(self): """Return a list of all encoded key names.""" @@ -241,14 +261,17 @@ class FileCache(MutableMapping): else: return set(file_keys + list(self._buffer)) - def _write_to_file(self, filename, bytesvalue): + def __write_to_file(self, filename, value): """Write bytesvalue to filename.""" fh, tmp = tempfile.mkstemp() with os.fdopen(fh, self._flag) as f: - f.write(self._dumps(bytesvalue)) + f.write(value) rename(tmp, filename) os.chmod(filename, self._mode) + def _write_to_file(self, filename, bytesvalue): + self.__write_to_file(filename, self._dumps(bytesvalue)) + def _read_from_file(self, filename): """Read data from filename.""" try: @@ -265,6 +288,7 @@ class FileCache(MutableMapping): else: filename = self._key_to_filename(ekey) self._write_to_file(filename, value) + self.__write_to_file(filename + self.key_file_ext, ekey) def __getitem__(self, key): ekey = self._encode_key(key) @@ -274,8 +298,9 @@ class FileCache(MutableMapping): except KeyError: pass filename = self._key_to_filename(ekey) - if filename not in self._all_filenames(): + if not os.path.isfile(filename): raise KeyError(key) + return self._read_from_file(filename) def __delitem__(self, key): @@ -292,6 +317,11 @@ class FileCache(MutableMapping): except (IOError, OSError): pass + try: + os.remove(filename + self.key_file_ext) + except (IOError, OSError): + pass + def __iter__(self): for key in self._all_keys(): yield self._decode_key(key) @@ -301,4 +331,10 @@ class FileCache(MutableMapping): def __contains__(self, key): ekey = self._encode_key(key) - return ekey in self._all_keys() + if not self._sync: + try: + return ekey in self._buffer + except KeyError: + pass + filename = self._key_to_filename(ekey) + return os.path.isfile(filename) diff --git a/libs/subliminal_patch/providers/legendastv.py b/libs/subliminal_patch/providers/legendastv.py index cab6867d6..e92b91993 100644 --- a/libs/subliminal_patch/providers/legendastv.py +++ b/libs/subliminal_patch/providers/legendastv.py @@ -199,7 +199,7 @@ class LegendasTVProvider(_LegendasTVProvider): # attempt to get the releases from the cache cache_key = releases_key.format(archive_id=a.id, archive_name=a.name) - releases = str(region.get(cache_key, expiration_time=expiration_time)) + releases = region.get(cache_key, expiration_time=expiration_time) # the releases are not in cache or cache is expired if releases == NO_VALUE: @@ -226,7 +226,7 @@ class LegendasTVProvider(_LegendasTVProvider): releases.append(name) # cache the releases - region.set(cache_key, bytearray(releases, encoding='utf-8')) + region.set(cache_key, bytearray(releases)) # iterate over releases for r in releases: diff --git a/views/wanted.tpl b/views/wanted.tpl index c21628974..5b05edb40 100644 --- a/views/wanted.tpl +++ b/views/wanted.tpl @@ -38,6 +38,7 @@ % from database import TableEpisodes, TableMovies, System % import operator % from config import settings + % from functools import reduce %episodes_missing_subtitles_clause = [ % (TableEpisodes.missing_subtitles != '[]')