From a8683132a77771bf764a20827ac9a8d73fea56da Mon Sep 17 00:00:00 2001 From: Eric Reed Date: Sun, 3 Mar 2013 12:07:13 -0500 Subject: [PATCH 1/2] Create LastfmBaseProvider --- .../MediaBrowser.Controller.csproj | 1 + .../Providers/Music/LastfmBaseProvider.cs | 365 ++++++++++++++++++ .../Entities/MetadataProviders.cs | 6 +- 3 files changed, 371 insertions(+), 1 deletion(-) create mode 100644 MediaBrowser.Controller/Providers/Music/LastfmBaseProvider.cs diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj index 839cf2f404..80c44d50c4 100644 --- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj +++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj @@ -134,6 +134,7 @@ + diff --git a/MediaBrowser.Controller/Providers/Music/LastfmBaseProvider.cs b/MediaBrowser.Controller/Providers/Music/LastfmBaseProvider.cs new file mode 100644 index 0000000000..7e99d684f0 --- /dev/null +++ b/MediaBrowser.Controller/Providers/Music/LastfmBaseProvider.cs @@ -0,0 +1,365 @@ +using System.Net; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Entities.Audio; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Logging; +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Model.Serialization; + +namespace MediaBrowser.Controller.Providers.Music +{ + class LastfmProviderException : ApplicationException + { + public LastfmProviderException(string msg) + : base(msg) + { + } + + } + /// + /// Class MovieDbProvider + /// + public abstract class LastfmBaseProvider : BaseMetadataProvider + { + /// + /// Gets the json serializer. + /// + /// The json serializer. + protected IJsonSerializer JsonSerializer { get; private set; } + + /// + /// Gets the HTTP client. + /// + /// The HTTP client. + protected IHttpClient HttpClient { get; private set; } + + /// + /// The name of the local json meta file for this item type + /// + protected string LocalMetaFileName { get; set; } + + /// + /// Initializes a new instance of the class. + /// + /// The json serializer. + /// The HTTP client. + /// The Log manager + /// jsonSerializer + public LastfmBaseProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient, ILogManager logManager) + : base(logManager) + { + if (jsonSerializer == null) + { + throw new ArgumentNullException("jsonSerializer"); + } + if (httpClient == null) + { + throw new ArgumentNullException("httpClient"); + } + JsonSerializer = jsonSerializer; + HttpClient = httpClient; + } + + /// + /// Gets the priority. + /// + /// The priority. + public override MetadataProviderPriority Priority + { + get { return MetadataProviderPriority.Second; } + } + + /// + /// Supportses the specified item. + /// + /// The item. + /// true if XXXX, false otherwise + public override bool Supports(BaseItem item) + { + return item is MusicArtist; + } + + /// + /// Gets a value indicating whether [requires internet]. + /// + /// true if [requires internet]; otherwise, false. + public override bool RequiresInternet + { + get + { + return true; + } + } + + /// + /// If we save locally, refresh if they delete something + /// + protected override bool RefreshOnFileSystemStampChange + { + get + { + return Kernel.Instance.Configuration.SaveLocalMeta; + } + } + + protected const string RootUrl = @"http://ws.audioscrobbler.com/2.0/"; + protected static string ApiKey = "7b76553c3eb1d341d642755aecc40a33"; + + static readonly Regex[] NameMatches = new[] { + new Regex(@"(?.*)\((?\d{4})\)"), // matches "My Movie (2001)" and gives us the name and the year + new Regex(@"(?.*)") // last resort matches the whole string as the name + }; + + protected override bool NeedsRefreshInternal(BaseItem item, BaseProviderInfo providerInfo) + { + if (item.DontFetchMeta) return false; + + if (Kernel.Instance.Configuration.SaveLocalMeta && HasFileSystemStampChanged(item, providerInfo)) + { + //If they deleted something from file system, chances are, this item was mis-identified the first time + item.SetProviderId(MetadataProviders.Musicbrainz, null); + Logger.Debug("LastfmProvider reports file system stamp change..."); + return true; + + } + + if (providerInfo.LastRefreshStatus == ProviderRefreshStatus.CompletedWithErrors) + { + Logger.Debug("LastfmProvider for {0} - last attempt had errors. Will try again.", item.Path); + return true; + } + + var downloadDate = providerInfo.LastRefreshed; + + if (Kernel.Instance.Configuration.MetadataRefreshDays == -1 && downloadDate != DateTime.MinValue) + { + return false; + } + + if (DateTime.Today.Subtract(item.DateCreated).TotalDays > 180 && downloadDate != DateTime.MinValue) + return false; // don't trigger a refresh data for item that are more than 6 months old and have been refreshed before + + if (DateTime.Today.Subtract(downloadDate).TotalDays < Kernel.Instance.Configuration.MetadataRefreshDays) // only refresh every n days + return false; + + + Logger.Debug("LastfmProvider - " + item.Name + " needs refresh. Download date: " + downloadDate + " item created date: " + item.DateCreated + " Check for Update age: " + Kernel.Instance.Configuration.MetadataRefreshDays); + return true; + } + + /// + /// Fetches metadata and returns true or false indicating if any work that requires persistence was done + /// + /// The item. + /// if set to true [force]. + /// The cancellation token + /// Task{System.Boolean}. + protected override async Task FetchAsyncInternal(BaseItem item, bool force, CancellationToken cancellationToken) + { + if (item.DontFetchMeta) + { + Logger.Info("LastfmProvider - Not fetching because requested to ignore " + item.Name); + return false; + } + + cancellationToken.ThrowIfCancellationRequested(); + + if (!Kernel.Instance.Configuration.SaveLocalMeta || !HasLocalMeta(item) || (force && !HasLocalMeta(item))) + { + try + { + await FetchData(item, cancellationToken).ConfigureAwait(false); + SetLastRefreshed(item, DateTime.UtcNow); + } + catch (LastfmProviderException) + { + SetLastRefreshed(item, DateTime.UtcNow, ProviderRefreshStatus.CompletedWithErrors); + } + + return true; + } + Logger.Debug("LastfmProvider not fetching because local meta exists for " + item.Name); + SetLastRefreshed(item, DateTime.UtcNow); + return true; + } + + /// + /// Determines whether [has local meta] [the specified item]. + /// + /// The item. + /// true if [has local meta] [the specified item]; otherwise, false. + private bool HasLocalMeta(BaseItem item) + { + return item.ResolveArgs.ContainsMetaFileByName(LocalMetaFileName); + } + + /// + /// Fetches the items data. + /// + /// The item. + /// + /// Task. + protected abstract Task FetchData(BaseItem item, CancellationToken cancellationToken); + + /// + /// Parses the name. + /// + /// The name. + /// Name of the just. + /// The year. + protected void ParseName(string name, out string justName, out int? year) + { + justName = null; + year = null; + foreach (var re in NameMatches) + { + Match m = re.Match(name); + if (m.Success) + { + justName = m.Groups["name"].Value.Trim(); + string y = m.Groups["year"] != null ? m.Groups["year"].Value : null; + int temp; + year = Int32.TryParse(y, out temp) ? temp : (int?)null; + break; + } + } + } + + /// + /// Encodes an URL. + /// + /// The name. + /// System.String. + protected static string UrlEncode(string name) + { + return WebUtility.UrlEncode(name); + } + + /// + /// The remove + /// + const string remove = "\"'!`?"; + // "Face/Off" support. + /// + /// The spacers + /// + const string spacers = "/,.:;\\(){}[]+-_=–*"; // (there are not actually two - in the they are different char codes) + /// + /// The replace start numbers + /// + static readonly Dictionary ReplaceStartNumbers = new Dictionary { + {"1 ","one "}, + {"2 ","two "}, + {"3 ","three "}, + {"4 ","four "}, + {"5 ","five "}, + {"6 ","six "}, + {"7 ","seven "}, + {"8 ","eight "}, + {"9 ","nine "}, + {"10 ","ten "}, + {"11 ","eleven "}, + {"12 ","twelve "}, + {"13 ","thirteen "}, + {"100 ","one hundred "}, + {"101 ","one hundred one "} + }; + + /// + /// The replace end numbers + /// + static readonly Dictionary ReplaceEndNumbers = new Dictionary { + {" 1"," i"}, + {" 2"," ii"}, + {" 3"," iii"}, + {" 4"," iv"}, + {" 5"," v"}, + {" 6"," vi"}, + {" 7"," vii"}, + {" 8"," viii"}, + {" 9"," ix"}, + {" 10"," x"} + }; + + /// + /// Gets the name of the comparable. + /// + /// The name. + /// The logger. + /// System.String. + internal static string GetComparableName(string name, ILogger logger) + { + name = name.ToLower(); + name = name.Replace("á", "a"); + name = name.Replace("é", "e"); + name = name.Replace("í", "i"); + name = name.Replace("ó", "o"); + name = name.Replace("ú", "u"); + name = name.Replace("ü", "u"); + name = name.Replace("ñ", "n"); + foreach (var pair in ReplaceStartNumbers) + { + if (name.StartsWith(pair.Key)) + { + name = name.Remove(0, pair.Key.Length); + name = pair.Value + name; + logger.Info("MovieDbProvider - Replaced Start Numbers: " + name); + } + } + foreach (var pair in ReplaceEndNumbers) + { + if (name.EndsWith(pair.Key)) + { + name = name.Remove(name.IndexOf(pair.Key), pair.Key.Length); + name = name + pair.Value; + logger.Info("MovieDbProvider - Replaced End Numbers: " + name); + } + } + name = name.Normalize(NormalizationForm.FormKD); + var sb = new StringBuilder(); + foreach (var c in name) + { + if (c >= 0x2B0 && c <= 0x0333) + { + // skip char modifier and diacritics + } + else if (remove.IndexOf(c) > -1) + { + // skip chars we are removing + } + else if (spacers.IndexOf(c) > -1) + { + sb.Append(" "); + } + else if (c == '&') + { + sb.Append(" and "); + } + else + { + sb.Append(c); + } + } + name = sb.ToString(); + name = name.Replace(", the", ""); + name = name.Replace(" the ", " "); + name = name.Replace("the ", ""); + + string prevName; + do + { + prevName = name; + name = name.Replace(" ", " "); + } while (name.Length != prevName.Length); + + return name.Trim(); + } + + } +} diff --git a/MediaBrowser.Model/Entities/MetadataProviders.cs b/MediaBrowser.Model/Entities/MetadataProviders.cs index e5324e1e3d..28e4b1646a 100644 --- a/MediaBrowser.Model/Entities/MetadataProviders.cs +++ b/MediaBrowser.Model/Entities/MetadataProviders.cs @@ -21,6 +21,10 @@ namespace MediaBrowser.Model.Entities /// /// The tvcom /// - Tvcom + Tvcom, + /// + /// MusicBrainz + /// + Musicbrainz } } From 77c894d228f4be978ff09adba8b6f0b7db943f5b Mon Sep 17 00:00:00 2001 From: Eric Reed Date: Sun, 3 Mar 2013 12:47:03 -0500 Subject: [PATCH 2/2] Some stuff that doesn't need to be in LastfmBase --- .../Providers/Music/LastfmBaseProvider.cs | 163 ------------------ 1 file changed, 163 deletions(-) diff --git a/MediaBrowser.Controller/Providers/Music/LastfmBaseProvider.cs b/MediaBrowser.Controller/Providers/Music/LastfmBaseProvider.cs index 7e99d684f0..364986130f 100644 --- a/MediaBrowser.Controller/Providers/Music/LastfmBaseProvider.cs +++ b/MediaBrowser.Controller/Providers/Music/LastfmBaseProvider.cs @@ -1,13 +1,9 @@ using System.Net; using MediaBrowser.Common.Net; using MediaBrowser.Controller.Entities; -using MediaBrowser.Controller.Entities.Audio; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; using System; -using System.Collections.Generic; -using System.Text; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using MediaBrowser.Model.Serialization; @@ -75,16 +71,6 @@ namespace MediaBrowser.Controller.Providers.Music get { return MetadataProviderPriority.Second; } } - /// - /// Supportses the specified item. - /// - /// The item. - /// true if XXXX, false otherwise - public override bool Supports(BaseItem item) - { - return item is MusicArtist; - } - /// /// Gets a value indicating whether [requires internet]. /// @@ -111,11 +97,6 @@ namespace MediaBrowser.Controller.Providers.Music protected const string RootUrl = @"http://ws.audioscrobbler.com/2.0/"; protected static string ApiKey = "7b76553c3eb1d341d642755aecc40a33"; - static readonly Regex[] NameMatches = new[] { - new Regex(@"(?.*)\((?\d{4})\)"), // matches "My Movie (2001)" and gives us the name and the year - new Regex(@"(?.*)") // last resort matches the whole string as the name - }; - protected override bool NeedsRefreshInternal(BaseItem item, BaseProviderInfo providerInfo) { if (item.DontFetchMeta) return false; @@ -207,30 +188,6 @@ namespace MediaBrowser.Controller.Providers.Music /// Task. protected abstract Task FetchData(BaseItem item, CancellationToken cancellationToken); - /// - /// Parses the name. - /// - /// The name. - /// Name of the just. - /// The year. - protected void ParseName(string name, out string justName, out int? year) - { - justName = null; - year = null; - foreach (var re in NameMatches) - { - Match m = re.Match(name); - if (m.Success) - { - justName = m.Groups["name"].Value.Trim(); - string y = m.Groups["year"] != null ? m.Groups["year"].Value : null; - int temp; - year = Int32.TryParse(y, out temp) ? temp : (int?)null; - break; - } - } - } - /// /// Encodes an URL. /// @@ -241,125 +198,5 @@ namespace MediaBrowser.Controller.Providers.Music return WebUtility.UrlEncode(name); } - /// - /// The remove - /// - const string remove = "\"'!`?"; - // "Face/Off" support. - /// - /// The spacers - /// - const string spacers = "/,.:;\\(){}[]+-_=–*"; // (there are not actually two - in the they are different char codes) - /// - /// The replace start numbers - /// - static readonly Dictionary ReplaceStartNumbers = new Dictionary { - {"1 ","one "}, - {"2 ","two "}, - {"3 ","three "}, - {"4 ","four "}, - {"5 ","five "}, - {"6 ","six "}, - {"7 ","seven "}, - {"8 ","eight "}, - {"9 ","nine "}, - {"10 ","ten "}, - {"11 ","eleven "}, - {"12 ","twelve "}, - {"13 ","thirteen "}, - {"100 ","one hundred "}, - {"101 ","one hundred one "} - }; - - /// - /// The replace end numbers - /// - static readonly Dictionary ReplaceEndNumbers = new Dictionary { - {" 1"," i"}, - {" 2"," ii"}, - {" 3"," iii"}, - {" 4"," iv"}, - {" 5"," v"}, - {" 6"," vi"}, - {" 7"," vii"}, - {" 8"," viii"}, - {" 9"," ix"}, - {" 10"," x"} - }; - - /// - /// Gets the name of the comparable. - /// - /// The name. - /// The logger. - /// System.String. - internal static string GetComparableName(string name, ILogger logger) - { - name = name.ToLower(); - name = name.Replace("á", "a"); - name = name.Replace("é", "e"); - name = name.Replace("í", "i"); - name = name.Replace("ó", "o"); - name = name.Replace("ú", "u"); - name = name.Replace("ü", "u"); - name = name.Replace("ñ", "n"); - foreach (var pair in ReplaceStartNumbers) - { - if (name.StartsWith(pair.Key)) - { - name = name.Remove(0, pair.Key.Length); - name = pair.Value + name; - logger.Info("MovieDbProvider - Replaced Start Numbers: " + name); - } - } - foreach (var pair in ReplaceEndNumbers) - { - if (name.EndsWith(pair.Key)) - { - name = name.Remove(name.IndexOf(pair.Key), pair.Key.Length); - name = name + pair.Value; - logger.Info("MovieDbProvider - Replaced End Numbers: " + name); - } - } - name = name.Normalize(NormalizationForm.FormKD); - var sb = new StringBuilder(); - foreach (var c in name) - { - if (c >= 0x2B0 && c <= 0x0333) - { - // skip char modifier and diacritics - } - else if (remove.IndexOf(c) > -1) - { - // skip chars we are removing - } - else if (spacers.IndexOf(c) > -1) - { - sb.Append(" "); - } - else if (c == '&') - { - sb.Append(" and "); - } - else - { - sb.Append(c); - } - } - name = sb.ToString(); - name = name.Replace(", the", ""); - name = name.Replace(" the ", " "); - name = name.Replace("the ", ""); - - string prevName; - do - { - prevName = name; - name = name.Replace(" ", " "); - } while (name.Length != prevName.Length); - - return name.Trim(); - } - } }