Merge remote-tracking branch 'origin/development' into development

libs/cfscrape/__init__.py

@@ -77,18 +77,18 @@ class CloudflareScraper(Session):
             pass
 
     def request(self, method, url, *args, **kwargs):
-        self.headers = (
-            OrderedDict(
-                [
-                    ('User-Agent', self.headers['User-Agent']),
-                    ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),
-                    ('Accept-Language', 'en-US,en;q=0.5'),
-                    ('Accept-Encoding', 'gzip, deflate'),
-                    ('Connection',  'close'),
-                    ('Upgrade-Insecure-Requests', '1')
-                ]
-            )
-        )
+        if not isinstance(self.headers, OrderedDict):
+            self.headers = \
+                OrderedDict(
+                    [
+                        ('User-Agent', self.headers['User-Agent']),
+                        ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),
+                        ('Accept-Language', 'en-US,en;q=0.5'),
+                        ('Accept-Encoding', 'gzip, deflate'),
+                        ('Connection',  'close'),
+                        ('Upgrade-Insecure-Requests', '1')
+                    ]
+                )
 
         resp = super(CloudflareScraper, self).request(method, url, *args, **kwargs)
 
@@ -127,7 +127,7 @@ class CloudflareScraper(Session):
         submit_url = '{}://{}/cdn-cgi/l/chk_jschl'.format(parsed_url.scheme, domain)
 
         cloudflare_kwargs = deepcopy(original_kwargs)
-        headers = cloudflare_kwargs.setdefault('headers', {'Referer': resp.url})
+        headers = cloudflare_kwargs.setdefault('headers', OrderedDict({'Referer': resp.url}))
 
         try:
             params = cloudflare_kwargs.setdefault(

libs/subliminal_patch/http.py

@@ -1,4 +1,5 @@
 # coding=utf-8
+
 import certifi
 import ssl
 import os
@@ -8,7 +9,9 @@ import requests
 import xmlrpclib
 import dns.resolver
 
+from collections import OrderedDict
 from requests import exceptions
+from requests.utils import default_user_agent
 from urllib3.util import connection
 from retry.api import retry_call
 from exceptions import APIThrottled
@@ -61,13 +64,18 @@ class CertifiSession(TimeoutSession):
 class CFSession(CloudflareScraper, CertifiSession, TimeoutSession):
     def __init__(self):
         super(CFSession, self).__init__()
-        self.headers.update({
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-            'Accept-Language': 'en-US,en;q=0.5',
-            'Cache-Control': 'no-cache',
-            'Pragma': 'no-cache',
-            'DNT': '1'
-        })
+        self.headers = OrderedDict([
+            ('User-Agent', default_user_agent()),
+            ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),
+            ('Accept-Language', 'en-US,en;q=0.5'),
+            ('Accept-Encoding', 'gzip, deflate'),
+            ('Connection', 'keep-alive'),
+            ('Pragma', 'no-cache'),
+            ('Cache-Control', 'no-cache'),
+            ('Upgrade-Insecure-Requests', '1'),
+            ('DNT', '1'),
+        ])
+        self.debug = os.environ.get("CF_DEBUG", False)
 
     def request(self, method, url, *args, **kwargs):
         parsed_url = urlparse(url)

libs/subliminal_patch/providers/subscene.py

@@ -128,7 +128,6 @@ class SubsceneProvider(Provider, ProviderSubtitleArchiveMixin):
         self.session = Session()
         from .utils import FIRST_THOUSAND_OR_SO_USER_AGENTS as AGENT_LIST
         self.session.headers['User-Agent'] = AGENT_LIST[randint(0, len(AGENT_LIST) - 1)]
-        self.session.headers['Referer'] = "https://subscene.com"
 
     def terminate(self):
         logger.info("Closing session")