diff --git a/libs/cfscrape/__init__.py b/libs/cfscrape/__init__.py index 57afe4312..e1d0c8fc8 100644 --- a/libs/cfscrape/__init__.py +++ b/libs/cfscrape/__init__.py @@ -77,18 +77,18 @@ class CloudflareScraper(Session): pass def request(self, method, url, *args, **kwargs): - self.headers = ( - OrderedDict( - [ - ('User-Agent', self.headers['User-Agent']), - ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'), - ('Accept-Language', 'en-US,en;q=0.5'), - ('Accept-Encoding', 'gzip, deflate'), - ('Connection', 'close'), - ('Upgrade-Insecure-Requests', '1') - ] - ) - ) + if not isinstance(self.headers, OrderedDict): + self.headers = \ + OrderedDict( + [ + ('User-Agent', self.headers['User-Agent']), + ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'), + ('Accept-Language', 'en-US,en;q=0.5'), + ('Accept-Encoding', 'gzip, deflate'), + ('Connection', 'close'), + ('Upgrade-Insecure-Requests', '1') + ] + ) resp = super(CloudflareScraper, self).request(method, url, *args, **kwargs) @@ -127,7 +127,7 @@ class CloudflareScraper(Session): submit_url = '{}://{}/cdn-cgi/l/chk_jschl'.format(parsed_url.scheme, domain) cloudflare_kwargs = deepcopy(original_kwargs) - headers = cloudflare_kwargs.setdefault('headers', {'Referer': resp.url}) + headers = cloudflare_kwargs.setdefault('headers', OrderedDict({'Referer': resp.url})) try: params = cloudflare_kwargs.setdefault( diff --git a/libs/subliminal_patch/http.py b/libs/subliminal_patch/http.py index 8683c6eda..84fa3f2c4 100644 --- a/libs/subliminal_patch/http.py +++ b/libs/subliminal_patch/http.py @@ -1,4 +1,5 @@ # coding=utf-8 + import certifi import ssl import os @@ -8,7 +9,9 @@ import requests import xmlrpclib import dns.resolver +from collections import OrderedDict from requests import exceptions +from requests.utils import default_user_agent from urllib3.util import connection from retry.api import retry_call from exceptions import APIThrottled @@ -61,13 +64,18 @@ class CertifiSession(TimeoutSession): class CFSession(CloudflareScraper, CertifiSession, TimeoutSession): def __init__(self): super(CFSession, self).__init__() - self.headers.update({ - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.5', - 'Cache-Control': 'no-cache', - 'Pragma': 'no-cache', - 'DNT': '1' - }) + self.headers = OrderedDict([ + ('User-Agent', default_user_agent()), + ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'), + ('Accept-Language', 'en-US,en;q=0.5'), + ('Accept-Encoding', 'gzip, deflate'), + ('Connection', 'keep-alive'), + ('Pragma', 'no-cache'), + ('Cache-Control', 'no-cache'), + ('Upgrade-Insecure-Requests', '1'), + ('DNT', '1'), + ]) + self.debug = os.environ.get("CF_DEBUG", False) def request(self, method, url, *args, **kwargs): parsed_url = urlparse(url) diff --git a/libs/subliminal_patch/providers/subscene.py b/libs/subliminal_patch/providers/subscene.py index 7f2758b14..1e2a2a2c8 100644 --- a/libs/subliminal_patch/providers/subscene.py +++ b/libs/subliminal_patch/providers/subscene.py @@ -128,7 +128,6 @@ class SubsceneProvider(Provider, ProviderSubtitleArchiveMixin): self.session = Session() from .utils import FIRST_THOUSAND_OR_SO_USER_AGENTS as AGENT_LIST self.session.headers['User-Agent'] = AGENT_LIST[randint(0, len(AGENT_LIST) - 1)] - self.session.headers['Referer'] = "https://subscene.com" def terminate(self): logger.info("Closing session")