using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.Threading; using Newtonsoft.Json; using NLog; using NzbDrone.Common.Cache; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Common.Serializer; using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Parser { public interface IFingerprintingService { bool IsSetup(); Version FpcalcVersion(); void Lookup(List tracks, double threshold); } public class AcoustId { public double Duration { get; set; } public string Fingerprint { get; set; } } public class FingerprintingService : IFingerprintingService { private const string _acoustIdUrl = "https://api.acoustid.org/v2/lookup"; private const string _acoustIdApiKey = "QANd68ji1L"; private const int _fingerprintingTimeout = 10000; private readonly Logger _logger; private readonly IHttpClient _httpClient; private readonly IHttpRequestBuilderFactory _customerRequestBuilder; private readonly ICached _cache; private readonly string _fpcalcPath; private readonly Version _fpcalcVersion; private readonly string _fpcalcArgs; public FingerprintingService(Logger logger, IHttpClient httpClient, ICacheManager cacheManager) { _logger = logger; _httpClient = httpClient; _cache = cacheManager.GetCache(GetType()); _customerRequestBuilder = new HttpRequestBuilder(_acoustIdUrl).CreateFactory(); // An exception here will cause Lidarr to fail to start, so catch any errors try { _fpcalcPath = GetFpcalcPath(); if (_fpcalcPath.IsNotNullOrWhiteSpace()) { _fpcalcVersion = GetFpcalcVersion(); _fpcalcArgs = GetFpcalcArgs(); } } catch (Exception ex) { _logger.Error(ex, "Something went wrong detecting fpcalc"); } } public bool IsSetup() => _fpcalcPath.IsNotNullOrWhiteSpace(); public Version FpcalcVersion() => _fpcalcVersion; private string GetFpcalcPath() { string path = null; // Take the fpcalc from the install directory if it exists path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "fpcalc"); if (OsInfo.IsWindows) { path += ".exe"; } if (File.Exists(path)) { return path; } // Otherwise search path for a candidate and check it works if (OsInfo.IsLinux) { // must be on users path on Linux path = "fpcalc"; // check that the command exists Process p = new Process(); p.StartInfo.FileName = path; p.StartInfo.Arguments = "-version"; p.StartInfo.UseShellExecute = false; p.StartInfo.CreateNoWindow = true; p.StartInfo.RedirectStandardOutput = true; try { p.Start(); // To avoid deadlocks, always read the output stream first and then wait. string output = p.StandardOutput.ReadToEnd(); p.WaitForExit(1000); } catch { _logger.Debug("fpcalc not found"); return null; } return path; } else { // on OSX / Windows, we have put fpcalc in the application folder _logger.Warn("fpcalc missing from application directory"); return null; } } private Version GetFpcalcVersion() { if (_fpcalcPath == null) { return null; } Process p = new Process(); p.StartInfo.FileName = _fpcalcPath; p.StartInfo.Arguments = $"-version"; p.StartInfo.UseShellExecute = false; p.StartInfo.CreateNoWindow = true; p.StartInfo.RedirectStandardOutput = true; p.Start(); // To avoid deadlocks, always read the output stream first and then wait. string output = p.StandardOutput.ReadToEnd(); p.WaitForExit(1000); if (p.ExitCode != 0) { _logger.Warn("Could not get fpcalc version (may be known issue with fpcalc v1.4)"); return null; } var versionstring = Regex.Match(output, @"\d\.\d\.\d").Value; if (versionstring.IsNullOrWhiteSpace()) { return null; } var version = new Version(versionstring); _logger.Debug($"fpcalc version: {version}"); return version; } private string GetFpcalcArgs() { var args = ""; if (_fpcalcVersion == null) { return args; } if (_fpcalcVersion >= new Version("1.4.0")) { args = "-json"; } if (_fpcalcVersion >= new Version("1.4.3")) { args += " -ignore-errors"; } return args; } public AcoustId ParseFpcalcJsonOutput(string output) { return Json.Deserialize(output); } public AcoustId ParseFpcalcTextOutput(string output) { var durationstring = Regex.Match(output, @"(?<=DURATION=)[\d\.]+(?=\s)").Value; if (durationstring.IsNullOrWhiteSpace() || !double.TryParse(durationstring, out var duration)) { return null; } var fingerprint = Regex.Match(output, @"(?<=FINGERPRINT=)[^\s]+").Value; if (fingerprint.IsNullOrWhiteSpace()) { return null; } return new AcoustId { Duration = duration, Fingerprint = fingerprint }; } public AcoustId ParseFpcalcOutput(string output) { if (output.IsNullOrWhiteSpace()) { return null; } if (_fpcalcArgs.Contains("-json")) { return ParseFpcalcJsonOutput(output); } else { return ParseFpcalcTextOutput(output); } } public AcoustId GetFingerprint(string file) { return _cache.Get(file, () => GetFingerprintUncached(file), TimeSpan.FromMinutes(30)); } private AcoustId GetFingerprintUncached(string file) { if (IsSetup() && File.Exists(file)) { Process p = new Process(); p.StartInfo.FileName = _fpcalcPath; p.StartInfo.Arguments = $"{_fpcalcArgs} \"{file}\""; p.StartInfo.UseShellExecute = false; p.StartInfo.CreateNoWindow = true; p.StartInfo.RedirectStandardOutput = true; p.StartInfo.RedirectStandardError = true; _logger.Trace("Executing {0} {1}", p.StartInfo.FileName, p.StartInfo.Arguments); StringBuilder output = new StringBuilder(); StringBuilder error = new StringBuilder(); // see https://stackoverflow.com/questions/139593/processstartinfo-hanging-on-waitforexit-why?lq=1 // this is most likely overkill... using (AutoResetEvent outputWaitHandle = new AutoResetEvent(false)) { using (AutoResetEvent errorWaitHandle = new AutoResetEvent(false)) { DataReceivedEventHandler outputHandler = (sender, e) => { if (e.Data == null) { outputWaitHandle.Set(); } else { output.AppendLine(e.Data); } }; DataReceivedEventHandler errorHandler = (sender, e) => { if (e.Data == null) { errorWaitHandle.Set(); } else { error.AppendLine(e.Data); } }; p.OutputDataReceived += outputHandler; p.ErrorDataReceived += errorHandler; p.Start(); p.BeginOutputReadLine(); p.BeginErrorReadLine(); if (p.WaitForExit(_fingerprintingTimeout) && outputWaitHandle.WaitOne(_fingerprintingTimeout) && errorWaitHandle.WaitOne(_fingerprintingTimeout)) { // Process completed. if (p.ExitCode != 0) { _logger.Warn($"fpcalc error: {error}"); return null; } else { return ParseFpcalcOutput(output.ToString()); } } else { // Timed out. Remove handlers to avoid object disposed error p.OutputDataReceived -= outputHandler; p.ErrorDataReceived -= errorHandler; _logger.Warn($"fpcalc timed out. {error}"); return null; } } } } return null; } public void Lookup(List tracks, double threshold) { if (!IsSetup()) { return; } Lookup(tracks.Select(x => Tuple.Create(x, GetFingerprint(x.Path))).ToList(), threshold); } public void Lookup(List> files, double threshold) { var toLookup = files.Where(x => x.Item2 != null).ToList(); if (!toLookup.Any()) { return; } var request = GenerateRequest(toLookup); var response = GetResponse(request); ParseResponse(response, toLookup, threshold); } public HttpRequest GenerateRequest(List> toLookup) { var httpRequest = _customerRequestBuilder.Create() .WithRateLimit(0.334) .Build(); var sb = new StringBuilder($"client={_acoustIdApiKey}&format=json&meta=recordingids&batch=1", 2000); for (int i = 0; i < toLookup.Count; i++) { sb.Append($"&duration.{i}={toLookup[i].Item2.Duration:F0}&fingerprint.{i}={toLookup[i].Item2.Fingerprint}"); } // they prefer a gzipped body httpRequest.SetContent(Encoding.UTF8.GetBytes(sb.ToString()).Compress()); httpRequest.Headers.Add("Content-Encoding", "gzip"); httpRequest.Headers.ContentType = "application/x-www-form-urlencoded"; httpRequest.SuppressHttpError = true; httpRequest.RequestTimeout = TimeSpan.FromSeconds(5); return httpRequest; } public LookupResponse GetResponse(HttpRequest request, int retry = 3) { HttpResponse httpResponse; try { httpResponse = _httpClient.Post(request); } catch (UnexpectedHtmlContentException e) { _logger.Warn(e, "AcoustId API gave invalid response"); return retry > 0 ? GetResponse(request, retry - 1) : null; } catch (Exception e) { _logger.Warn(e, "AcoustId API lookup failed"); return null; } var response = httpResponse.Resource; if (httpResponse.HasHttpError || (response != null && response.Status != "ok")) { if (response?.Error != null) { if (response.Error.Code == AcoustIdErrorCode.TooManyRequests && retry > 0) { _logger.Trace($"Too many requests, retrying in 1s"); Thread.Sleep(TimeSpan.FromSeconds(1)); return GetResponse(request, retry - 1); } _logger.Debug($"Webservice error {response.Error.Code}: {response.Error.Message}"); } else { _logger.Warn("HTTP Error - {0}", httpResponse); } return null; } return response; } private void ParseResponse(LookupResponse response, List> toLookup, double threshold) { if (response == null) { return; } // The API will give errors if fingerprint isn't found or is invalid. // We don't want to stop the entire import because the fingerprinting failed // so just log and return. foreach (var fileResponse in response.Fingerprints) { if (fileResponse.Results.Count == 0) { _logger.Debug("No results for given fingerprint."); continue; } foreach (var result in fileResponse.Results.Where(x => x.Recordings != null)) { _logger.Trace("Found: {0}, {1}, {2}", result.Id, result.Score, string.Join(", ", result.Recordings.Select(x => x.Id))); } var ids = fileResponse.Results.Where(x => x.Score > threshold && x.Recordings != null).SelectMany(y => y.Recordings.Select(z => z.Id)).Distinct().ToList(); _logger.Trace("All recordings: {0}", string.Join("\n", ids)); toLookup[fileResponse.index].Item1.AcoustIdResults = ids; } _logger.Debug("Fingerprinting complete."); var serializerSettings = Json.GetSerializerSettings(); serializerSettings.Formatting = Formatting.None; var output = new { Fingerprints = toLookup.Select(x => new { Path = x.Item1.Path, AcoustIdResults = x.Item1.AcoustIdResults }) }; _logger.Debug($"*** FingerprintingService TestCaseGenerator ***\n{JsonConvert.SerializeObject(output, serializerSettings)}"); } public class LookupResponse { public string Status { get; set; } public LookupError Error { get; set; } public List Fingerprints { get; set; } } public enum AcoustIdErrorCode { // https://github.com/acoustid/acoustid-server/blob/f671339ad9ab049c4d4361d3eadb6660a8fe4dda/acoustid/api/errors.py#L10 UnknownFormat = 1, MissingParameter = 2, InvalidFingerprint = 3, InvalidApikey = 4, Internal = 5, InvalidUserApikey = 6, InvalidUuid = 7, InvalidDuration = 8, InvalidBitrate = 9, InvalidForeignid = 10, InvalidMaxDurationDiff = 11, NotAllowed = 12, ServiceUnavailable = 13, TooManyRequests = 14, InvalidMusicbrainzAccessToken = 15, InsecureRequest = 16, UnknownApplication = 17, FingerprintNotFound = 18 } public class LookupError { public string Message { get; set; } public AcoustIdErrorCode Code { get; set; } } public class LookupResultListItem { public int index { get; set; } public List Results { get; set; } } public class LookupResult { public string Id { get; set; } public double Score { get; set; } public List Recordings { get; set; } } public class RecordingResult { public string Id { get; set; } } } }