New: Will now provider a clearer error if the indexer returned unexpected html content (indicating a blocked site) instead of failing with a parser error.

src/NzbDrone.Core.Test/IndexerTests/SeasonSearchFixture.cs

@@ -24,7 +24,9 @@ namespace NzbDrone.Core.Test.IndexerTests
         {
             _series = Builder<Series>.CreateNew().Build();
 
-            Mocker.GetMock<IHttpProvider>().Setup(s => s.DownloadString(It.IsAny<String>())).Returns("<xml></xml>");
+            var response = new HttpResponse(null, new HttpHeader(), "<xml></xml>", System.Net.HttpStatusCode.OK);
+            Mocker.GetMock<IHttpClient>()
+                .Setup(s => s.Get(It.IsAny<HttpRequest>())).Returns(response);
         }
 
         private IndexerBase<TestIndexerSettings> WithIndexer(bool paging, int resultCount)
@@ -57,7 +59,7 @@ namespace NzbDrone.Core.Test.IndexerTests
             var indexer = WithIndexer(true, 25);
             Subject.Fetch(indexer, new SeasonSearchCriteria { Series = _series, SceneTitles = new List<string>{_series.Title} });
 
-            Mocker.GetMock<IHttpProvider>().Verify(v => v.DownloadString(It.IsAny<String>()), Times.Once());
+            Mocker.GetMock<IHttpClient>().Verify(v => v.Get(It.IsAny<HttpRequest>()), Times.Once());
         }
 
         [Test]
@@ -66,7 +68,7 @@ namespace NzbDrone.Core.Test.IndexerTests
             var indexer = WithIndexer(false, 125);
             Subject.Fetch(indexer, new SeasonSearchCriteria { Series = _series, SceneTitles = new List<string> { _series.Title } });
 
-            Mocker.GetMock<IHttpProvider>().Verify(v => v.DownloadString(It.IsAny<String>()), Times.Once());
+            Mocker.GetMock<IHttpClient>().Verify(v => v.Get(It.IsAny<HttpRequest>()), Times.Once());
         }
 
         [Test]
@@ -75,7 +77,7 @@ namespace NzbDrone.Core.Test.IndexerTests
             var indexer = WithIndexer(true, 100);
             Subject.Fetch(indexer, new SeasonSearchCriteria { Series = _series, SceneTitles = new List<string> { _series.Title } });
 
-            Mocker.GetMock<IHttpProvider>().Verify(v => v.DownloadString(It.IsAny<String>()), Times.Exactly(10));
+            Mocker.GetMock<IHttpClient>().Verify(v => v.Get(It.IsAny<HttpRequest>()), Times.Exactly(10));
         }
     }
 

src/NzbDrone.Core/Indexers/IndexerFetchService.cs

@@ -24,11 +24,11 @@ namespace NzbDrone.Core.Indexers
     public class FetchFeedService : IFetchFeedFromIndexers
     {
         private readonly Logger _logger;
-        private readonly IHttpProvider _httpProvider;
+        private readonly IHttpClient _httpClient;
 
-        public FetchFeedService(IHttpProvider httpProvider, Logger logger)
+        public FetchFeedService(IHttpClient httpClient, Logger logger)
         {
-            _httpProvider = httpProvider;
+            _httpClient = httpClient;
             _logger = logger;
         }
 
@@ -140,7 +140,16 @@ namespace NzbDrone.Core.Indexers
                 try
                 {
                     _logger.Debug("Downloading Feed " + url);
-                    var xml = _httpProvider.DownloadString(url);
+                    var request = new HttpRequest(url);
+                    request.Headers.Accept = "text/xml, text/rss+xml, application/rss+xml";
+                    var response = _httpClient.Get(request);
+
+                    if (response.Headers.ContentType != null && response.Headers.ContentType.Split(';')[0] == "text/html")
+                    {
+                        throw new WebException("Indexer responded with html content. Site is likely blocked or unavailable.");
+                    }
+
+                    var xml = response.Content;
                     if (!string.IsNullOrWhiteSpace(xml))
                     {
                         result.AddRange(indexer.Parser.Process(xml, url));