You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Lidarr/src/NzbDrone.Core/Parser/FingerprintingService.cs

509 lines
17 KiB

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using Newtonsoft.Json;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Common.Serializer;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Parser
{
public interface IFingerprintingService
{
bool IsSetup();
Version FpcalcVersion();
void Lookup(List<LocalTrack> tracks, double threshold);
}
public class AcoustId
{
public double Duration { get; set; }
public string Fingerprint { get; set; }
}
public class FingerprintingService : IFingerprintingService
{
private const string _acoustIdUrl = "https://api.acoustid.org/v2/lookup";
private const string _acoustIdApiKey = "QANd68ji1L";
private const int _fingerprintingTimeout = 10000;
private readonly Logger _logger;
private readonly IHttpClient _httpClient;
private readonly IHttpRequestBuilderFactory _customerRequestBuilder;
private readonly ICached<AcoustId> _cache;
private readonly string _fpcalcPath;
private readonly Version _fpcalcVersion;
private readonly string _fpcalcArgs;
public FingerprintingService(Logger logger,
IHttpClient httpClient,
ICacheManager cacheManager)
{
_logger = logger;
_httpClient = httpClient;
_cache = cacheManager.GetCache<AcoustId>(GetType());
_customerRequestBuilder = new HttpRequestBuilder(_acoustIdUrl).CreateFactory();
// An exception here will cause Lidarr to fail to start, so catch any errors
try
{
_fpcalcPath = GetFpcalcPath();
if (_fpcalcPath.IsNotNullOrWhiteSpace())
{
_fpcalcVersion = GetFpcalcVersion();
_fpcalcArgs = GetFpcalcArgs();
}
}
catch (Exception ex)
{
_logger.Error(ex, "Something went wrong detecting fpcalc");
}
}
public bool IsSetup() => _fpcalcPath.IsNotNullOrWhiteSpace();
public Version FpcalcVersion() => _fpcalcVersion;
private string GetFpcalcPath()
{
string path = null;
// Take the fpcalc from the install directory if it exists
path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "fpcalc");
if (OsInfo.IsWindows)
{
path += ".exe";
}
if (File.Exists(path))
{
return path;
}
// Otherwise search path for a candidate and check it works
if (OsInfo.IsLinux)
{
// must be on users path on Linux
path = "fpcalc";
// check that the command exists
Process p = new Process();
p.StartInfo.FileName = path;
p.StartInfo.Arguments = "-version";
p.StartInfo.UseShellExecute = false;
p.StartInfo.CreateNoWindow = true;
p.StartInfo.RedirectStandardOutput = true;
try
{
p.Start();
// To avoid deadlocks, always read the output stream first and then wait.
string output = p.StandardOutput.ReadToEnd();
p.WaitForExit(1000);
}
catch
{
_logger.Debug("fpcalc not found");
return null;
}
return path;
}
else
{
// on OSX / Windows, we have put fpcalc in the application folder
_logger.Warn("fpcalc missing from application directory");
return null;
}
}
private Version GetFpcalcVersion()
{
if (_fpcalcPath == null)
{
return null;
}
Process p = new Process();
p.StartInfo.FileName = _fpcalcPath;
p.StartInfo.Arguments = $"-version";
p.StartInfo.UseShellExecute = false;
p.StartInfo.CreateNoWindow = true;
p.StartInfo.RedirectStandardOutput = true;
p.Start();
// To avoid deadlocks, always read the output stream first and then wait.
string output = p.StandardOutput.ReadToEnd();
p.WaitForExit(1000);
if (p.ExitCode != 0)
{
_logger.Warn("Could not get fpcalc version (may be known issue with fpcalc v1.4)");
return null;
}
var versionstring = Regex.Match(output, @"\d\.\d\.\d").Value;
if (versionstring.IsNullOrWhiteSpace())
{
return null;
}
var version = new Version(versionstring);
_logger.Debug($"fpcalc version: {version}");
return version;
}
private string GetFpcalcArgs()
{
var args = "";
if (_fpcalcVersion == null)
{
return args;
}
if (_fpcalcVersion >= new Version("1.4.0"))
{
args = "-json";
}
if (_fpcalcVersion >= new Version("1.4.3"))
{
args += " -ignore-errors";
}
return args;
}
public AcoustId ParseFpcalcJsonOutput(string output)
{
return Json.Deserialize<AcoustId>(output);
}
public AcoustId ParseFpcalcTextOutput(string output)
{
var durationstring = Regex.Match(output, @"(?<=DURATION=)[\d\.]+(?=\s)").Value;
double duration;
if (durationstring.IsNullOrWhiteSpace() || !double.TryParse(durationstring, out duration))
{
return null;
}
var fingerprint = Regex.Match(output, @"(?<=FINGERPRINT=)[^\s]+").Value;
if (fingerprint.IsNullOrWhiteSpace())
{
return null;
}
return new AcoustId
{
Duration = duration,
Fingerprint = fingerprint
};
}
public AcoustId ParseFpcalcOutput(string output)
{
if (output.IsNullOrWhiteSpace())
{
return null;
}
if (_fpcalcArgs.Contains("-json"))
{
return ParseFpcalcJsonOutput(output);
}
else
{
return ParseFpcalcTextOutput(output);
}
}
public AcoustId GetFingerprint(string file)
{
return _cache.Get(file, () => GetFingerprintUncached(file), TimeSpan.FromMinutes(30));
}
private AcoustId GetFingerprintUncached(string file)
{
if (IsSetup() && File.Exists(file))
{
Process p = new Process();
p.StartInfo.FileName = _fpcalcPath;
p.StartInfo.Arguments = $"{_fpcalcArgs} \"{file}\"";
p.StartInfo.UseShellExecute = false;
p.StartInfo.CreateNoWindow = true;
p.StartInfo.RedirectStandardOutput = true;
p.StartInfo.RedirectStandardError = true;
_logger.Trace("Executing {0} {1}", p.StartInfo.FileName, p.StartInfo.Arguments);
StringBuilder output = new StringBuilder();
StringBuilder error = new StringBuilder();
// see https://stackoverflow.com/questions/139593/processstartinfo-hanging-on-waitforexit-why?lq=1
// this is most likely overkill...
using (AutoResetEvent outputWaitHandle = new AutoResetEvent(false))
{
using (AutoResetEvent errorWaitHandle = new AutoResetEvent(false))
{
DataReceivedEventHandler outputHandler = (sender, e) =>
{
if (e.Data == null)
{
outputWaitHandle.Set();
}
else
{
output.AppendLine(e.Data);
}
};
DataReceivedEventHandler errorHandler = (sender, e) =>
{
if (e.Data == null)
{
errorWaitHandle.Set();
}
else
{
error.AppendLine(e.Data);
}
};
p.OutputDataReceived += outputHandler;
p.ErrorDataReceived += errorHandler;
p.Start();
p.BeginOutputReadLine();
p.BeginErrorReadLine();
if (p.WaitForExit(_fingerprintingTimeout) &&
outputWaitHandle.WaitOne(_fingerprintingTimeout) &&
errorWaitHandle.WaitOne(_fingerprintingTimeout))
{
// Process completed.
if (p.ExitCode != 0)
{
_logger.Warn($"fpcalc error: {error}");
return null;
}
else
{
return ParseFpcalcOutput(output.ToString());
}
}
else
{
// Timed out. Remove handlers to avoid object disposed error
p.OutputDataReceived -= outputHandler;
p.ErrorDataReceived -= errorHandler;
_logger.Warn($"fpcalc timed out. {error}");
return null;
}
}
}
}
return null;
}
public void Lookup(List<LocalTrack> tracks, double threshold)
{
if (!IsSetup())
{
return;
}
Lookup(tracks.Select(x => Tuple.Create(x, GetFingerprint(x.Path))).ToList(), threshold);
}
public void Lookup(List<Tuple<LocalTrack, AcoustId>> files, double threshold)
{
var toLookup = files.Where(x => x.Item2 != null).ToList();
if (!toLookup.Any())
{
return;
}
var request = GenerateRequest(toLookup);
var response = GetResponse(request);
ParseResponse(response, toLookup, threshold);
}
public HttpRequest GenerateRequest(List<Tuple<LocalTrack, AcoustId>> toLookup)
{
var httpRequest = _customerRequestBuilder.Create()
.WithRateLimit(0.334)
.Build();
var sb = new StringBuilder($"client={_acoustIdApiKey}&format=json&meta=recordingids&batch=1", 2000);
for (int i = 0; i < toLookup.Count; i++)
{
sb.Append($"&duration.{i}={toLookup[i].Item2.Duration:F0}&fingerprint.{i}={toLookup[i].Item2.Fingerprint}");
}
// they prefer a gzipped body
httpRequest.SetContent(Encoding.UTF8.GetBytes(sb.ToString()).Compress());
httpRequest.Headers.Add("Content-Encoding", "gzip");
httpRequest.Headers.ContentType = "application/x-www-form-urlencoded";
httpRequest.SuppressHttpError = true;
httpRequest.RequestTimeout = TimeSpan.FromSeconds(5);
return httpRequest;
}
public LookupResponse GetResponse(HttpRequest request, int retry = 3)
{
HttpResponse<LookupResponse> httpResponse;
try
{
httpResponse = _httpClient.Post<LookupResponse>(request);
}
catch (UnexpectedHtmlContentException e)
{
_logger.Warn(e, "AcoustId API gave invalid response");
return retry > 0 ? GetResponse(request, retry - 1) : null;
}
catch (Exception e)
{
_logger.Warn(e, "AcoustId API lookup failed");
return null;
}
var response = httpResponse.Resource;
if (httpResponse.HasHttpError || (response != null && response.Status != "ok"))
{
if (response?.Error != null)
{
if (response.Error.Code == AcoustIdErrorCode.TooManyRequests && retry > 0)
{
_logger.Trace($"Too many requests, retrying in 1s");
Thread.Sleep(TimeSpan.FromSeconds(1));
return GetResponse(request, retry - 1);
}
_logger.Debug($"Webservice error {response.Error.Code}: {response.Error.Message}");
}
else
{
_logger.Warn("HTTP Error - {0}", httpResponse);
}
return null;
}
return response;
}
private void ParseResponse(LookupResponse response, List<Tuple<LocalTrack, AcoustId>> toLookup, double threshold)
{
if (response == null)
{
return;
}
// The API will give errors if fingerprint isn't found or is invalid.
// We don't want to stop the entire import because the fingerprinting failed
// so just log and return.
foreach (var fileResponse in response.Fingerprints)
{
if (fileResponse.Results.Count == 0)
{
_logger.Debug("No results for given fingerprint.");
continue;
}
foreach (var result in fileResponse.Results.Where(x => x.Recordings != null))
{
_logger.Trace("Found: {0}, {1}, {2}", result.Id, result.Score, string.Join(", ", result.Recordings.Select(x => x.Id)));
}
var ids = fileResponse.Results.Where(x => x.Score > threshold && x.Recordings != null).SelectMany(y => y.Recordings.Select(z => z.Id)).Distinct().ToList();
_logger.Trace("All recordings: {0}", string.Join("\n", ids));
toLookup[fileResponse.index].Item1.AcoustIdResults = ids;
}
_logger.Debug("Fingerprinting complete.");
var serializerSettings = Json.GetSerializerSettings();
serializerSettings.Formatting = Formatting.None;
var output = new { Fingerprints = toLookup.Select(x => new { Path = x.Item1.Path, AcoustIdResults = x.Item1.AcoustIdResults }) };
_logger.Debug($"*** FingerprintingService TestCaseGenerator ***\n{JsonConvert.SerializeObject(output, serializerSettings)}");
}
public class LookupResponse
{
public string Status { get; set; }
public LookupError Error { get; set; }
public List<LookupResultListItem> Fingerprints { get; set; }
}
public enum AcoustIdErrorCode
{
// https://github.com/acoustid/acoustid-server/blob/f671339ad9ab049c4d4361d3eadb6660a8fe4dda/acoustid/api/errors.py#L10
UnknownFormat = 1,
MissingParameter = 2,
InvalidFingerprint = 3,
InvalidApikey = 4,
Internal = 5,
InvalidUserApikey = 6,
InvalidUuid = 7,
InvalidDuration = 8,
InvalidBitrate = 9,
InvalidForeignid = 10,
InvalidMaxDurationDiff = 11,
NotAllowed = 12,
ServiceUnavailable = 13,
TooManyRequests = 14,
InvalidMusicbrainzAccessToken = 15,
InsecureRequest = 16,
UnknownApplication = 17,
FingerprintNotFound = 18
}
public class LookupError
{
public string Message { get; set; }
public AcoustIdErrorCode Code { get; set; }
}
public class LookupResultListItem
{
public int index { get; set; }
public List<LookupResult> Results { get; set; }
}
public class LookupResult
{
public string Id { get; set; }
public double Score { get; set; }
public List<RecordingResult> Recordings { get; set; }
}
public class RecordingResult
{
public string Id { get; set; }
}
}
}