using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using AngleSharp.Dom;
using AngleSharp.Html;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
using AngleSharp.Text;
using Newtonsoft.Json.Linq;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Indexers.Definitions.Cardigann.Exceptions;
using NzbDrone.Core.IndexerSearch.Definitions;
using NzbDrone.Core.Parser;
using NzbDrone.Core.ThingiProvider;

namespace NzbDrone.Core.Indexers.Definitions.Cardigann
{
    public class CardigannRequestGenerator : CardigannBase, IIndexerRequestGenerator
    {
        public IIndexerHttpClient HttpClient { get; set; }
        public ProviderDefinition Definition { get; set; }
        public IDictionary<string, string> Cookies { get; set; }
        protected HttpResponse landingResult;
        protected IHtmlDocument landingResultDocument;
        protected override string SiteLink => ResolveSiteLink();

        private readonly TimeSpan _rateLimit;

        public CardigannRequestGenerator(IConfigService configService,
                                         CardigannDefinition definition,
                                         Logger logger,
                                         TimeSpan rateLimit)
        : base(configService, definition, logger)
        {
            _rateLimit = rateLimit;
        }

        public Func<IDictionary<string, string>> GetCookies { get; set; }
        public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }

        public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
        {
            _logger.Trace("Getting Movie search");

            var pageableRequests = new IndexerPageableRequestChain();

            var variables = GetQueryVariableDefaults(searchCriteria);

            variables[".Query.Movie"] = null;
            variables[".Query.Year"] = searchCriteria.Year?.ToString() ?? null;
            variables[".Query.Genre"] = searchCriteria.Genre;
            variables[".Query.IMDBID"] = searchCriteria.FullImdbId;
            variables[".Query.IMDBIDShort"] = searchCriteria.ImdbId;
            variables[".Query.TMDBID"] = searchCriteria.TmdbId?.ToString() ?? null;
            variables[".Query.TraktID"] = searchCriteria.TraktId?.ToString() ?? null;
            variables[".Query.DoubanID"] = searchCriteria.DoubanId?.ToString() ?? null;

            var request = GetRequest(variables, searchCriteria);

            if (bool.TryParse(ApplyGoTemplateText(_definition.Search.Pageable, variables), out _))
            {
                pageableRequests.AddTier(request);
            }
            else
            {
                pageableRequests.Add(request);
            }

            return pageableRequests;
        }

        public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria)
        {
            _logger.Trace("Getting Music search");

            var pageableRequests = new IndexerPageableRequestChain();

            var variables = GetQueryVariableDefaults(searchCriteria);

            variables[".Query.Album"] = searchCriteria.Album;
            variables[".Query.Artist"] = searchCriteria.Artist;
            variables[".Query.Label"] = searchCriteria.Label;
            variables[".Query.Genre"] = searchCriteria.Genre;
            variables[".Query.Year"] = searchCriteria.Year?.ToString() ?? null;
            variables[".Query.Track"] = searchCriteria.Track;

            var request = GetRequest(variables, searchCriteria);

            if (bool.TryParse(ApplyGoTemplateText(_definition.Search.Pageable, variables), out _))
            {
                pageableRequests.AddTier(request);
            }
            else
            {
                pageableRequests.Add(request);
            }

            return pageableRequests;
        }

        public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria)
        {
            _logger.Trace("Getting TV search");

            var pageableRequests = new IndexerPageableRequestChain();

            var variables = GetQueryVariableDefaults(searchCriteria);

            variables[".Query.Series"] = null;
            variables[".Query.Ep"] = searchCriteria.Episode;
            variables[".Query.Season"] = searchCriteria.Season?.ToString() ?? null;
            variables[".Query.Genre"] = searchCriteria.Genre;
            variables[".Query.Year"] = searchCriteria.Year?.ToString() ?? null;
            variables[".Query.IMDBID"] = searchCriteria.FullImdbId;
            variables[".Query.IMDBIDShort"] = searchCriteria.ImdbId;
            variables[".Query.TVDBID"] = searchCriteria.TvdbId?.ToString() ?? null;
            variables[".Query.TMDBID"] = searchCriteria.TmdbId?.ToString() ?? null;
            variables[".Query.TVRageID"] = searchCriteria.RId?.ToString() ?? null;
            variables[".Query.TVMazeID"] = searchCriteria.TvMazeId?.ToString() ?? null;
            variables[".Query.TraktID"] = searchCriteria.TraktId?.ToString() ?? null;
            variables[".Query.DoubanID"] = searchCriteria.DoubanId?.ToString() ?? null;
            variables[".Query.Episode"] = searchCriteria.EpisodeSearchString;

            var request = GetRequest(variables, searchCriteria);

            if (bool.TryParse(ApplyGoTemplateText(_definition.Search.Pageable, variables), out _))
            {
                pageableRequests.AddTier(request);
            }
            else
            {
                pageableRequests.Add(request);
            }

            return pageableRequests;
        }

        public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria)
        {
            _logger.Trace("Getting Book search");

            var pageableRequests = new IndexerPageableRequestChain();

            var variables = GetQueryVariableDefaults(searchCriteria);

            variables[".Query.Author"] = searchCriteria.Author;
            variables[".Query.Title"] = searchCriteria.Title;
            variables[".Query.Genre"] = searchCriteria.Genre;
            variables[".Query.Publisher"] = searchCriteria.Publisher;
            variables[".Query.Year"] = searchCriteria.Year?.ToString() ?? null;

            var request = GetRequest(variables, searchCriteria);

            if (bool.TryParse(ApplyGoTemplateText(_definition.Search.Pageable, variables), out _))
            {
                pageableRequests.AddTier(request);
            }
            else
            {
                pageableRequests.Add(request);
            }

            return pageableRequests;
        }

        public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria)
        {
            _logger.Trace("Getting Basic search");

            var pageableRequests = new IndexerPageableRequestChain();

            var variables = GetQueryVariableDefaults(searchCriteria);

            var request = GetRequest(variables, searchCriteria);

            if ((_definition?.Search?.Pageable.IsNotNullOrWhiteSpace() ?? true) || bool.TryParse(ApplyGoTemplateText(_definition.Search.Pageable, variables), out _))
            {
                pageableRequests.AddTier(request);
            }
            else
            {
                pageableRequests.Add(request);
            }

            return pageableRequests;
        }

        private Dictionary<string, object> GetQueryVariableDefaults(SearchCriteriaBase searchCriteria)
        {
            var variables = GetBaseTemplateVariables();

            variables[".Query.Type"] = searchCriteria.SearchType;
            variables[".Query.Q"] = searchCriteria.SearchTerm;
            variables[".Query.Categories"] = searchCriteria.Categories;
            variables[".Query.Limit"] = GetQueryLimit(searchCriteria).ToString();
            variables[".Query.Offset"] = searchCriteria.Offset.ToString();
            variables[".Query.Extended"] = null;
            variables[".Query.APIKey"] = null;
            variables[".Query.Genre"] = null;

            //Movie
            variables[".Query.Movie"] = null;
            variables[".Query.Year"] = null;
            variables[".Query.IMDBID"] = null;
            variables[".Query.IMDBIDShort"] = null;
            variables[".Query.TMDBID"] = null;

            //Tv
            variables[".Query.Series"] = null;
            variables[".Query.Ep"] = null;
            variables[".Query.Season"] = null;
            variables[".Query.TVDBID"] = null;
            variables[".Query.TVRageID"] = null;
            variables[".Query.TVMazeID"] = null;
            variables[".Query.TraktID"] = null;
            variables[".Query.DoubanID"] = null;
            variables[".Query.Episode"] = null;

            //Music
            variables[".Query.Album"] = null;
            variables[".Query.Artist"] = null;
            variables[".Query.Label"] = null;
            variables[".Query.Track"] = null;

            //Book
            variables[".Query.Author"] = null;
            variables[".Query.Title"] = null;
            variables[".Query.Publisher"] = null;

            return variables;
        }

        public async Task DoLogin()
        {
            var login = _definition.Login;

            var variables = GetBaseTemplateVariables();
            var headers = ParseCustomHeaders(_definition.Login?.Headers ?? _definition.Search?.Headers, variables);

            if (login.Method == "post")
            {
                var pairs = new Dictionary<string, string>();

                if (login.Inputs != null && login.Inputs.Any())
                {
                    foreach (var input in login.Inputs)
                    {
                        var value = ApplyGoTemplateText(input.Value);
                        pairs.Add(input.Key, value);
                    }
                }

                var loginUrl = ResolvePath(login.Path).ToString();

                CookiesUpdater(null, null);

                var requestBuilder = new HttpRequestBuilder(loginUrl)
                {
                    LogResponseContent = true,
                    Method = HttpMethod.Post,
                    AllowAutoRedirect = true,
                    SuppressHttpError = true,
                    Encoding = _encoding
                };

                foreach (var pair in pairs)
                {
                    requestBuilder.AddFormParameter(pair.Key, pair.Value);
                }

                Cookies = null;
                if (login.Cookies != null)
                {
                    Cookies = CookieUtil.CookieHeaderToDictionary(string.Join("; ", login.Cookies));
                }

                var request = requestBuilder
                    .SetCookies(Cookies ?? new Dictionary<string, string>())
                    .SetHeaders(headers ?? new Dictionary<string, string>())
                    .SetHeader("Referer", SiteLink)
                    .WithRateLimit(_rateLimit.TotalSeconds)
                    .Build();

                var response = await HttpClient.ExecuteProxiedAsync(request, Definition);

                Cookies = response.GetCookies();

                CheckForError(response, login.Error);

                CookiesUpdater(Cookies, DateTime.Now.AddDays(30));
            }
            else if (login.Method == "form")
            {
                var loginUrl = ResolvePath(login.Path).ToString();

                var queryCollection = new NameValueCollection();
                var pairs = new Dictionary<string, string>();

                var formSelector = login.Form ?? "form";

                // landingResultDocument might not be initiated if the login is caused by a re-login during a query
                if (landingResultDocument == null)
                {
                    await GetConfigurationForSetup(true);
                }

                if (landingResultDocument == null)
                {
                    throw new CardigannConfigException(_definition, $"Login failed: Invalid document received from {loginUrl}");
                }

                var form = landingResultDocument.QuerySelector(formSelector);
                if (form == null)
                {
                    throw new CardigannConfigException(_definition, $"Login failed: No form found on {loginUrl} using form selector {formSelector}");
                }

                var inputs = form.QuerySelectorAll("input");
                if (inputs == null)
                {
                    throw new CardigannConfigException(_definition, $"Login failed: No inputs found on {loginUrl} using form selector {formSelector}");
                }

                var submitUrlstr = form.GetAttribute("action");
                if (login.Submitpath != null)
                {
                    submitUrlstr = login.Submitpath;
                }

                foreach (var input in inputs)
                {
                    var name = input.GetAttribute("name");

                    if (name == null || input.IsDisabled())
                    {
                        continue;
                    }

                    if (input is IHtmlInputElement element &&
                        element.Type.IsOneOf(InputTypeNames.Checkbox, InputTypeNames.Radio) &&
                        !input.IsChecked())
                    {
                        continue;
                    }

                    var value = input.GetAttribute("value") ?? "";

                    pairs[name] = value;
                }

                if (login.Inputs != null && login.Inputs.Any())
                {
                    foreach (var input in login.Inputs)
                    {
                        var value = ApplyGoTemplateText(input.Value);
                        var inputKey = input.Key;
                        if (login.Selectors)
                        {
                            var inputElement = landingResultDocument.QuerySelector(input.Key);
                            if (inputElement == null)
                            {
                                throw new CardigannConfigException(_definition, string.Format("Login failed: No input found using selector {0}", input.Key));
                            }

                            inputKey = inputElement.GetAttribute("name");
                        }

                        pairs[inputKey] = value;
                    }
                }

                // selector inputs
                if (login.Selectorinputs != null)
                {
                    foreach (var selectorinput in login.Selectorinputs)
                    {
                        string value = null;
                        try
                        {
                            value = HandleSelector(selectorinput.Value, landingResultDocument.FirstElementChild);
                            pairs[selectorinput.Key] = value;
                        }
                        catch (Exception ex)
                        {
                            throw new CardigannException(string.Format("Error while parsing selector input={0}, selector={1}, value={2}: {3}", selectorinput.Key, selectorinput.Value.Selector, value, ex.Message));
                        }
                    }
                }

                // getselector inputs
                if (login.Getselectorinputs != null)
                {
                    foreach (var selectorinput in login.Getselectorinputs)
                    {
                        string value = null;
                        try
                        {
                            value = HandleSelector(selectorinput.Value, landingResultDocument.FirstElementChild);
                            queryCollection[selectorinput.Key] = value;
                        }
                        catch (Exception ex)
                        {
                            throw new CardigannException(string.Format("Error while parsing get selector input={0}, selector={1}, value={2}: {3}", selectorinput.Key, selectorinput.Value.Selector, value, ex.Message));
                        }
                    }
                }

                if (queryCollection.Count > 0)
                {
                    submitUrlstr += "?" + queryCollection.GetQueryString();
                }

                var submitUrl = ResolvePath(submitUrlstr, new Uri(loginUrl));

                // automatically solve simpleCaptchas, if used
                var simpleCaptchaPresent = landingResultDocument.QuerySelector("script[src*=\"simpleCaptcha\"]");
                if (simpleCaptchaPresent != null)
                {
                    var captchaUrl = ResolvePath("simpleCaptcha.php?numImages=1");

                    var requestBuilder = new HttpRequestBuilder(captchaUrl.ToString())
                    {
                        LogResponseContent = true,
                        Method = HttpMethod.Get,
                        Encoding = _encoding
                    };

                    var request = requestBuilder
                        .SetCookies(Cookies)
                        .SetHeaders(headers ?? new Dictionary<string, string>())
                        .SetHeader("Referer", loginUrl)
                        .WithRateLimit(_rateLimit.TotalSeconds)
                        .Build();

                    var simpleCaptchaResult = await HttpClient.ExecuteProxiedAsync(request, Definition);

                    var simpleCaptchaJSON = JObject.Parse(simpleCaptchaResult.Content);
                    var captchaSelection = simpleCaptchaJSON["images"][0]["hash"].ToString();
                    pairs["captchaSelection"] = captchaSelection;
                    pairs["submitme"] = "X";
                }

                if (login.Captcha != null)
                {
                    var captcha = login.Captcha;

                    if (Settings.ExtraFieldData.TryGetValue("CAPTCHA", out var captchaText) && ((string)captchaText).IsNotNullOrWhiteSpace())
                    {
                        var input = captcha.Input;

                        if (login.Selectors)
                        {
                            var inputElement = landingResultDocument.QuerySelector(captcha.Input);

                            if (inputElement == null)
                            {
                                throw new CardigannConfigException(_definition, $"Login failed: No captcha input found using {captcha.Input}");
                            }

                            input = inputElement.GetAttribute("name");
                        }

                        pairs[input] = (string)captchaText;
                    }
                }

                // clear landingResults/Document, otherwise we might use an old version for a new relogin (if GetConfigurationForSetup() wasn't called before)
                landingResult = null;
                landingResultDocument = null;

                HttpResponse loginResult = null;
                var enctype = form.GetAttribute("enctype");
                if (enctype == "multipart/form-data")
                {
                    var boundary = "---------------------------" + DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1)).TotalSeconds.ToString().Replace(".", "");
                    var bodyParts = new List<string>();

                    foreach (var pair in pairs)
                    {
                        var part = "--" + boundary + "\r\n" +
                          "Content-Disposition: form-data; name=\"" + pair.Key + "\"\r\n" +
                          "\r\n" +
                          pair.Value;
                        bodyParts.Add(part);
                    }

                    bodyParts.Add("--" + boundary + "--");

                    headers.Add("Content-Type", "multipart/form-data; boundary=" + boundary);
                    var body = string.Join("\r\n", bodyParts);

                    var requestBuilder = new HttpRequestBuilder(submitUrl.ToString())
                    {
                        LogResponseContent = true,
                        Method = HttpMethod.Post,
                        AllowAutoRedirect = true,
                        Encoding = _encoding
                    };

                    foreach (var pair in pairs)
                    {
                        requestBuilder.AddFormParameter(pair.Key, pair.Value);
                    }

                    var request = requestBuilder
                        .SetCookies(Cookies)
                        .SetHeaders(headers ?? new Dictionary<string, string>())
                        .SetHeader("Referer", SiteLink)
                        .WithRateLimit(_rateLimit.TotalSeconds)
                        .Build();

                    request.SetContent(body);

                    loginResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
                }
                else
                {
                    var requestBuilder = new HttpRequestBuilder(submitUrl.ToString())
                    {
                        LogResponseContent = true,
                        Method = HttpMethod.Post,
                        AllowAutoRedirect = true,
                        SuppressHttpError = true,
                        Encoding = _encoding
                    };

                    foreach (var pair in pairs)
                    {
                        requestBuilder.AddFormParameter(pair.Key, pair.Value);
                    }

                    var request = requestBuilder
                        .SetCookies(Cookies)
                        .SetHeaders(headers ?? new Dictionary<string, string>())
                        .SetHeader("Referer", loginUrl)
                        .WithRateLimit(_rateLimit.TotalSeconds)
                        .Build();

                    loginResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
                }

                Cookies = loginResult.GetCookies();
                CheckForError(loginResult, login.Error);
                CookiesUpdater(Cookies, DateTime.Now.AddDays(30));
            }
            else if (login.Method == "cookie")
            {
                CookiesUpdater(null, null);
                Settings.ExtraFieldData.TryGetValue("cookie", out var cookies);
                CookiesUpdater(CookieUtil.CookieHeaderToDictionary((string)cookies), DateTime.Now.AddDays(30));
            }
            else if (login.Method == "get")
            {
                var queryCollection = new NameValueCollection();

                if (login.Inputs != null && login.Inputs.Any())
                {
                    foreach (var input in login.Inputs)
                    {
                        var value = ApplyGoTemplateText(input.Value);
                        queryCollection.Add(input.Key, value);
                    }
                }

                var loginUrl = ResolvePath(login.Path + "?" + queryCollection.GetQueryString()).ToString();

                CookiesUpdater(null, null);

                var requestBuilder = new HttpRequestBuilder(loginUrl)
                {
                    LogResponseContent = true,
                    Method = HttpMethod.Get,
                    SuppressHttpError = true,
                    Encoding = _encoding
                };

                var request = requestBuilder
                    .SetHeaders(headers ?? new Dictionary<string, string>())
                    .SetHeader("Referer", SiteLink)
                    .WithRateLimit(_rateLimit.TotalSeconds)
                    .Build();

                var response = await HttpClient.ExecuteProxiedAsync(request, Definition);

                Cookies = response.GetCookies();

                CheckForError(response, login.Error);

                CookiesUpdater(Cookies, DateTime.Now.AddDays(30));
            }
            else if (login.Method == "oneurl")
            {
                var oneUrl = ApplyGoTemplateText(login.Inputs["oneurl"]);
                var loginUrl = ResolvePath(login.Path + oneUrl).ToString();

                CookiesUpdater(null, null);

                var requestBuilder = new HttpRequestBuilder(loginUrl)
                {
                    LogResponseContent = true,
                    Method = HttpMethod.Get,
                    SuppressHttpError = true,
                    Encoding = _encoding
                };

                var request = requestBuilder
                    .SetHeaders(headers ?? new Dictionary<string, string>())
                    .SetHeader("Referer", SiteLink)
                    .WithRateLimit(_rateLimit.TotalSeconds)
                    .Build();

                var response = await HttpClient.ExecuteProxiedAsync(request, Definition);

                Cookies = response.GetCookies();

                CheckForError(response, login.Error);

                CookiesUpdater(Cookies, DateTime.Now.AddDays(30));
            }
            else
            {
                throw new NotImplementedException($"Login method {login.Method} not implemented");
            }
        }

        protected bool CheckForError(HttpResponse loginResult, IList<ErrorBlock> errorBlocks)
        {
            if (loginResult.StatusCode == HttpStatusCode.Unauthorized)
            {
                throw new HttpException(loginResult);
            }

            if (errorBlocks == null)
            {
                return true;
            }

            var resultParser = new HtmlParser();
            using var resultDocument = resultParser.ParseDocument(loginResult.Content);
            foreach (var error in errorBlocks)
            {
                var selection = resultDocument.QuerySelector(error.Selector);
                if (selection != null)
                {
                    var errorMessage = selection.TextContent;
                    if (error.Message != null)
                    {
                        errorMessage = HandleSelector(error.Message, resultDocument.FirstElementChild);
                    }

                    throw new CardigannConfigException(_definition, string.Format("Error: {0}", errorMessage.Trim()));
                }
            }

            return true;
        }

        public async Task<Captcha> GetConfigurationForSetup(bool automaticLogin)
        {
            var login = _definition.Login;

            if (login is not { Method: "form" })
            {
                return null;
            }

            var variables = GetBaseTemplateVariables();
            var headers = ParseCustomHeaders(_definition.Login?.Headers ?? _definition.Search?.Headers, variables);

            var loginUrl = ResolvePath(login.Path);

            var requestBuilder = new HttpRequestBuilder(loginUrl.AbsoluteUri)
            {
                LogResponseContent = true,
                Method = HttpMethod.Get,
                Encoding = _encoding
            };

            if (_definition.Followredirect)
            {
                requestBuilder.AllowAutoRedirect = true;
            }

            Cookies = null;
            if (login.Cookies != null)
            {
                Cookies = CookieUtil.CookieHeaderToDictionary(string.Join("; ", login.Cookies));
            }

            var request = requestBuilder
                .SetCookies(Cookies ?? new Dictionary<string, string>())
                .SetHeaders(headers ?? new Dictionary<string, string>())
                .SetHeader("Referer", SiteLink)
                .WithRateLimit(_rateLimit.TotalSeconds)
                .Build();

            landingResult = await HttpClient.ExecuteProxiedAsync(request, Definition);

            Cookies = landingResult.GetCookies();

            var htmlParser = new HtmlParser();
            landingResultDocument = htmlParser.ParseDocument(landingResult.Content);

            Captcha captcha = null;

            if (login.Captcha != null)
            {
                captcha = await GetCaptcha(login);
            }

            if (captcha != null && automaticLogin)
            {
                throw new CardigannConfigException(_definition, "Found captcha during automatic login, aborting");
            }

            return captcha;
        }

        private async Task<Captcha> GetCaptcha(LoginBlock login)
        {
            var captcha = login.Captcha;

            var variables = GetBaseTemplateVariables();
            var headers = ParseCustomHeaders(_definition.Login?.Headers ?? _definition.Search?.Headers, variables);

            if (captcha.Type == "image")
            {
                var captchaElement = landingResultDocument.QuerySelector(captcha.Selector);
                if (captchaElement != null)
                {
                    var loginUrl = ResolvePath(login.Path);
                    var captchaUrl = ResolvePath(captchaElement.GetAttribute("src"), loginUrl);

                    var request = new HttpRequestBuilder(captchaUrl.ToString())
                        .SetCookies(Cookies ?? new Dictionary<string, string>())
                        .SetHeaders(headers ?? new Dictionary<string, string>())
                        .SetHeader("Referer", loginUrl.AbsoluteUri)
                        .SetEncoding(_encoding)
                        .WithRateLimit(_rateLimit.TotalSeconds)
                        .Build();

                    var response = await HttpClient.ExecuteProxiedAsync(request, Definition);

                    if (response.GetCookies().Any())
                    {
                        Cookies = response.GetCookies();
                    }

                    return new Captcha
                    {
                        ContentType = response.Headers.ContentType,
                        ImageData = response.ResponseData
                    };
                }

                _logger.Debug("CardigannIndexer ({0}): No captcha image found", _definition.Id);
            }
            else
            {
                throw new NotImplementedException(string.Format("Captcha type \"{0}\" is not implemented", captcha.Type));
            }

            return null;
        }

        protected string GetRedirectDomainHint(string requestUrl, string redirectUrl)
        {
            var siteLinkUri = new HttpUri(SiteLink);
            var requestUri = new HttpUri(requestUrl);
            var redirectUri = new HttpUri(redirectUrl);

            if (requestUri.Host.StartsWith(siteLinkUri.Host) && !redirectUri.Host.StartsWith(siteLinkUri.Host))
            {
                return redirectUri.Scheme + "://" + redirectUri.Host + "/";
            }

            return null;
        }

        protected string GetRedirectDomainHint(HttpResponse result) => GetRedirectDomainHint(result.Request.Url.ToString(), result.RedirectUrl);

        protected async Task<HttpResponse> HandleRequest(RequestBlock request, Dictionary<string, object> variables = null, string referer = null)
        {
            var requestLinkStr = ResolvePath(ApplyGoTemplateText(request.Path, variables)).ToString();

            Dictionary<string, string> pairs = null;
            var queryCollection = new NameValueCollection();

            var method = HttpMethod.Get;
            if (string.Equals(request.Method, "post", StringComparison.OrdinalIgnoreCase))
            {
                method = HttpMethod.Post;
                pairs = new Dictionary<string, string>();
            }

            if (request.Inputs != null)
            {
                foreach (var input in request.Inputs)
                {
                    var value = ApplyGoTemplateText(input.Value, variables);
                    if (method == HttpMethod.Get)
                    {
                        queryCollection.Add(input.Key, value);
                    }
                    else if (method == HttpMethod.Post)
                    {
                        pairs.Add(input.Key, value);
                    }
                }
            }

            if (queryCollection.Count > 0)
            {
                if (!requestLinkStr.Contains('?'))
                {
                    requestLinkStr += "?";
                }

                requestLinkStr += queryCollection.GetQueryString(_encoding, separator: request.Queryseparator);
            }

            var httpRequestBuilder = new HttpRequestBuilder(requestLinkStr)
            {
                Method = method,
                Encoding = _encoding
            };

            // Add form data for POST requests
            if (method == HttpMethod.Post)
            {
                foreach (var param in pairs)
                {
                    httpRequestBuilder.AddFormParameter(param.Key, param.Value);
                }
            }

            var headers = ParseCustomHeaders(_definition.Download?.Headers ?? _definition.Search?.Headers, variables);

            var httpRequest = httpRequestBuilder
                .SetCookies(Cookies ?? new Dictionary<string, string>())
                .SetHeaders(headers ?? new Dictionary<string, string>())
                .SetHeader("Referer", referer)
                .WithRateLimit(_rateLimit.TotalSeconds)
                .Build();

            _logger.Debug("CardigannIndexer ({0}): handleRequest() httpRequest={1}", _definition.Id, httpRequest);

            var response = await HttpClient.ExecuteProxiedAsync(httpRequest, Definition);

            _logger.Debug("CardigannIndexer ({0}): handleRequest() remote server returned {1}", _definition.Id, response.StatusCode);

            return response;
        }

        public async Task<HttpRequest> DownloadRequest(Uri link)
        {
            Cookies = GetCookies();
            var method = HttpMethod.Get;

            var variables = GetBaseTemplateVariables();
            AddTemplateVariablesFromUri(variables, link, ".DownloadUri");
            var headers = ParseCustomHeaders(_definition.Download?.Headers ?? _definition.Search?.Headers, variables);

            if (_definition.Download != null)
            {
                var download = _definition.Download;

                HttpResponse response = null;

                var request = new HttpRequestBuilder(link.ToString())
                    .SetCookies(Cookies ?? new Dictionary<string, string>())
                    .SetHeaders(headers ?? new Dictionary<string, string>())
                    .SetEncoding(_encoding)
                    .WithRateLimit(_rateLimit.TotalSeconds)
                    .Build();

                request.AllowAutoRedirect = true;

                var beforeBlock = download.Before;
                if (beforeBlock != null)
                {
                    if (beforeBlock.Pathselector != null)
                    {
                        response = await HttpClient.ExecuteProxiedAsync(request, Definition);
                        beforeBlock.Path = MatchSelector(response, beforeBlock.Pathselector, variables);
                    }

                    response = await HandleRequest(beforeBlock, variables, link.ToString());
                }

                if (download.Method == "post")
                {
                    method = HttpMethod.Post;
                }

                if (download.Infohash != null)
                {
                    try
                    {
                        if (!download.Infohash.Usebeforeresponse || download.Before == null || response == null)
                        {
                            response = await HttpClient.ExecuteProxiedAsync(request, Definition);
                        }

                        var hash = MatchSelector(response, download.Infohash.Hash, variables);
                        if (hash == null)
                        {
                            throw new CardigannException("InfoHash selectors didn't match hash.");
                        }

                        var title = MatchSelector(response, download.Infohash.Title, variables);
                        if (title == null)
                        {
                            throw new CardigannException("InfoHash selectors didn't match title.");
                        }

                        var magnet = MagnetLinkBuilder.BuildPublicMagnetLink(hash, title);
                        var torrentLink = ResolvePath(magnet, link);

                        var hashDownloadRequest = new HttpRequestBuilder(torrentLink.AbsoluteUri)
                            .SetCookies(Cookies ?? new Dictionary<string, string>())
                            .SetHeaders(headers ?? new Dictionary<string, string>())
                            .SetEncoding(_encoding)
                            .Build();

                        hashDownloadRequest.Method = method;

                        return hashDownloadRequest;
                    }
                    catch (Exception)
                    {
                        _logger.Error("CardigannIndexer ({0}): Exception with InfoHash block with hashSelector {1} and titleSelector {2}",
                            _definition.Id,
                            download.Infohash.Hash.Selector,
                            download.Infohash.Title.Selector);
                    }
                }
                else if (download.Selectors != null && download.Selectors.Any())
                {
                    foreach (var selector in download.Selectors)
                    {
                        var querySelector = ApplyGoTemplateText(selector.Selector, variables);

                        try
                        {
                            if (!selector.Usebeforeresponse || download.Before == null || response == null)
                            {
                                response = await HttpClient.ExecuteProxiedAsync(request, Definition);
                            }

                            var href = MatchSelector(response, selector, variables, debugMatch: true);

                            if (href == null)
                            {
                                continue;
                            }

                            var torrentLink = ResolvePath(href, link);

                            if (torrentLink.Scheme != "magnet" && _definition.TestLinkTorrent)
                            {
                                // Test link
                                var testLinkRequest = new HttpRequestBuilder(torrentLink.ToString())
                                    {
                                        AllowAutoRedirect = true
                                    }
                                    .SetCookies(Cookies ?? new Dictionary<string, string>())
                                    .SetHeaders(headers ?? new Dictionary<string, string>())
                                    .SetEncoding(_encoding)
                                    .WithRateLimit(_rateLimit.TotalSeconds)
                                    .Build();

                                response = await HttpClient.ExecuteProxiedAsync(testLinkRequest, Definition);

                                var content = response.Content;

                                if (content.Length >= 1 && content[0] != 'd')
                                {
                                    _logger.Debug("{0}: Download selector {1}'s torrent file is invalid, retrying with next available selector", _definition.Id, querySelector);

                                    continue;
                                }
                            }

                            link = torrentLink;

                            var selectorDownloadRequest = new HttpRequestBuilder(link.AbsoluteUri)
                                .SetCookies(Cookies ?? new Dictionary<string, string>())
                                .SetHeaders(headers ?? new Dictionary<string, string>())
                                .SetEncoding(_encoding)
                                .WithRateLimit(_rateLimit.TotalSeconds)
                                .Build();

                            selectorDownloadRequest.Method = method;

                            return selectorDownloadRequest;
                        }
                        catch (Exception ex)
                        {
                            _logger.Error(ex, "{0}: An exception occurred while trying selector {1}, retrying with next available selector.", _definition.Id, querySelector);
                        }
                    }

                    throw new CardigannException($"Download selectors didn't match for {link}");
                }
            }

            var downloadRequest = new HttpRequestBuilder(link.AbsoluteUri)
                .SetCookies(Cookies ?? new Dictionary<string, string>())
                .SetHeaders(headers ?? new Dictionary<string, string>())
                .SetEncoding(_encoding)
                .WithRateLimit(_rateLimit.TotalSeconds)
                .Build();

            downloadRequest.Method = method;

            return downloadRequest;
        }

        protected string MatchSelector(HttpResponse response, SelectorField selector, Dictionary<string, object> variables, bool debugMatch = false)
        {
            var selectorText = ApplyGoTemplateText(selector.Selector, variables);
            var parser = new HtmlParser();

            using var resultDocument = parser.ParseDocument(response.Content);

            var element = resultDocument.QuerySelector(selectorText);
            if (element == null)
            {
                _logger.Debug($"CardigannIndexer ({_definition.Id}): Selector {selectorText} could not match any elements.");
                return null;
            }

            if (debugMatch)
            {
                _logger.Debug($"CardigannIndexer ({_definition.Id}): Download selector {selector} matched:{element.ToHtmlPretty()}");
            }

            string val;
            if (selector.Attribute != null)
            {
                val = element.GetAttribute(selector.Attribute);
                if (val == null)
                {
                    throw new CardigannException($"Attribute \"{selector.Attribute}\" is not set for element {element.ToHtmlPretty()}");
                }
            }
            else
            {
                val = element.TextContent;
            }

            val = ApplyFilters(val, selector.Filters, variables);
            return val;
        }

        public bool CheckIfLoginIsNeeded(HttpResponse response)
        {
            if (_definition.Login == null)
            {
                return false;
            }

            if (response.HasHttpRedirect)
            {
                var domainHint = GetRedirectDomainHint(response);
                if (domainHint != null)
                {
                    var errormessage = "Got redirected to another domain. Try changing the indexer URL to " + domainHint + ".";

                    _logger.Warn(errormessage);
                }

                return true;
            }

            if (response.HasHttpError)
            {
                return true;
            }

            // Only run html test selector on html responses
            if (_definition.Login.Test?.Selector != null && (response.Headers.ContentType?.Contains("text/html") ?? true))
            {
                var parser = new HtmlParser();
                using var document = parser.ParseDocument(response.Content);

                var selection = document.QuerySelectorAll(_definition.Login.Test.Selector);
                if (selection.Length == 0)
                {
                    return true;
                }
            }

            return false;
        }

        private IEnumerable<IndexerRequest> GetRequest(Dictionary<string, object> variables, SearchCriteriaBase searchCriteria)
        {
            var search = _definition.Search;

            var mappedCategories = _categories.MapTorznabCapsToTrackers((int[])variables[".Query.Categories"]);
            if (mappedCategories.Count == 0)
            {
                mappedCategories = _defaultCategories;
            }

            var keywordTokens = new List<string>();
            var keywordTokenKeys = new List<string> { "Q", "Series", "Movie", "Year" };
            foreach (var key in keywordTokenKeys)
            {
                var value = (string)variables[".Query." + key];
                if (!string.IsNullOrWhiteSpace(value))
                {
                    keywordTokens.Add(value);
                }
            }

            if (!string.IsNullOrWhiteSpace((string)variables[".Query.Episode"]))
            {
                keywordTokens.Add((string)variables[".Query.Episode"]);
            }

            variables[".Query.Keywords"] = string.Join(" ", keywordTokens);
            variables[".Keywords"] = ApplyFilters((string)variables[".Query.Keywords"], search.Keywordsfilters, variables);

            if (search.PageSize == 0 && searchCriteria.Offset > 0)
            {
                // Indexer doesn't support pagination
                yield break;
            }

            var pageSize = Math.Min(GetQueryLimit(searchCriteria), search.PageSize);

            if (pageSize > 0)
            {
                variables[".PageSize"] = pageSize;
                variables[".Query.Page"] = (int)(searchCriteria.Offset / pageSize) + search.FirstPageNumber;
            }

            var searchUrls = new List<string>();

            foreach (var searchPath in search.Paths)
            {
                variables[".Categories"] = mappedCategories;

                // skip path if categories don't match
                if (searchPath.Categories != null && mappedCategories.Count > 0)
                {
                    var hasIntersect = mappedCategories.Intersect(searchPath.Categories).Any();

                    if (searchPath.Categories[0] == "!")
                    {
                        hasIntersect = !hasIntersect;
                    }

                    if (!hasIntersect)
                    {
                        variables[".Categories"] = mappedCategories.Except(searchPath.Categories).ToList();

                        continue;
                    }

                    variables[".Categories"] = mappedCategories.Intersect(searchPath.Categories).ToList();
                }

                // build search URL
                // HttpUtility.UrlPathEncode seems to only encode spaces, we use UrlEncode and replace + with %20 as a workaround
                var searchUrl = ResolvePath(ApplyGoTemplateText(searchPath.Path, variables, WebUtility.UrlEncode).Replace("+", "%20")).AbsoluteUri;
                var queryCollection = new List<KeyValuePair<string, string>>();
                var method = HttpMethod.Get;

                if (string.Equals(searchPath.Method, "post", StringComparison.OrdinalIgnoreCase))
                {
                    method = HttpMethod.Post;
                }

                var inputsList = new List<Dictionary<string, string>>();
                if (searchPath.Inheritinputs)
                {
                    inputsList.Add(search.Inputs);
                }

                inputsList.Add(searchPath.Inputs);

                foreach (var inputs in inputsList)
                {
                    if (inputs != null)
                    {
                        foreach (var input in inputs)
                        {
                            if (input.Key == "$raw")
                            {
                                var rawStr = ApplyGoTemplateText(input.Value, variables, WebUtility.UrlEncode);
                                foreach (var part in rawStr.Split('&'))
                                {
                                    var parts = part.Split(new[] { '=' }, 2);
                                    var key = parts[0];
                                    if (key.Length == 0)
                                    {
                                        continue;
                                    }

                                    var value = "";
                                    if (parts.Length == 2)
                                    {
                                        value = parts[1];
                                    }

                                    queryCollection.Add(key, value);
                                }
                            }
                            else
                            {
                                var inputValue = ApplyGoTemplateText(input.Value, variables);

                                if (inputValue.IsNotNullOrWhiteSpace() || search.AllowEmptyInputs)
                                {
                                    queryCollection.Add(input.Key, inputValue);
                                }
                            }
                        }
                    }
                }

                if (method == HttpMethod.Get && queryCollection.Count > 0)
                {
                    searchUrl += "?" + queryCollection.GetQueryString(_encoding);
                }

                if (method == HttpMethod.Get && searchUrls.Contains(searchUrl))
                {
                    _logger.Trace("Skip duplicated request {0}", searchUrl);

                    continue;
                }

                searchUrls.Add(searchUrl);

                _logger.Debug($"Adding request: {searchUrl}");

                var requestBuilder = new HttpRequestBuilder(searchUrl)
                {
                    Method = method,
                    Encoding = _encoding
                };

                // Add FormData for searchs that POST
                if (method == HttpMethod.Post)
                {
                    foreach (var param in queryCollection)
                    {
                        requestBuilder.AddFormParameter(param.Key, param.Value);
                    }
                }

                // send HTTP request
                if (search.Headers != null)
                {
                    var headers = ParseCustomHeaders(search.Headers, variables);
                    requestBuilder.SetHeaders(headers ?? new Dictionary<string, string>());
                }

                if (searchPath.Followredirect)
                {
                    requestBuilder.AllowAutoRedirect = true;
                }

                var request = requestBuilder
                    .WithRateLimit(_rateLimit.TotalSeconds)
                    .Build();

                yield return new CardigannRequest(request, variables, searchPath);
            }
        }

        private int GetQueryLimit(SearchCriteriaBase searchCriteria)
        {
            return Math.Min(Math.Min(searchCriteria.Limit, _definition?.Caps?.LimitsDefault ?? searchCriteria.Limit), _definition?.Caps?.LimitsMax ?? searchCriteria.Limit);
        }
    }
}