|
|
|
@ -38,12 +38,29 @@ custom_resolver = dns.resolver.Resolver(configure=False)
|
|
|
|
|
custom_resolver.nameservers = ['8.8.8.8', '1.1.1.1']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class CertifiSession(CloudflareScraper):
|
|
|
|
|
class TimeoutSession(requests.Session):
|
|
|
|
|
timeout = 10
|
|
|
|
|
|
|
|
|
|
def __init__(self, timeout=None):
|
|
|
|
|
super(TimeoutSession, self).__init__()
|
|
|
|
|
self.timeout = timeout or self.timeout
|
|
|
|
|
|
|
|
|
|
def request(self, method, url, *args, **kwargs):
|
|
|
|
|
if kwargs.get('timeout') is None:
|
|
|
|
|
kwargs['timeout'] = self.timeout
|
|
|
|
|
|
|
|
|
|
return super(TimeoutSession, self).request(method, url, *args, **kwargs)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class CertifiSession(TimeoutSession):
|
|
|
|
|
def __init__(self):
|
|
|
|
|
super(CertifiSession, self).__init__()
|
|
|
|
|
self.verify = pem_file
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class CFSession(CloudflareScraper, CertifiSession, TimeoutSession):
|
|
|
|
|
def __init__(self):
|
|
|
|
|
super(CFSession, self).__init__()
|
|
|
|
|
self.headers.update({
|
|
|
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
|
|
|
'Accept-Language': 'en-US,en;q=0.5',
|
|
|
|
@ -53,9 +70,6 @@ class CertifiSession(CloudflareScraper):
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
def request(self, method, url, *args, **kwargs):
|
|
|
|
|
if kwargs.get('timeout') is None:
|
|
|
|
|
kwargs['timeout'] = self.timeout
|
|
|
|
|
|
|
|
|
|
parsed_url = urlparse(url)
|
|
|
|
|
domain = parsed_url.netloc
|
|
|
|
|
|
|
|
|
@ -71,7 +85,7 @@ class CertifiSession(CloudflareScraper):
|
|
|
|
|
|
|
|
|
|
self.headers['User-Agent'] = user_agent
|
|
|
|
|
|
|
|
|
|
ret = super(CertifiSession, self).request(method, url, *args, **kwargs)
|
|
|
|
|
ret = super(CFSession, self).request(method, url, *args, **kwargs)
|
|
|
|
|
try:
|
|
|
|
|
cf_data = self.get_live_tokens(domain)
|
|
|
|
|
except:
|
|
|
|
@ -85,12 +99,11 @@ class CertifiSession(CloudflareScraper):
|
|
|
|
|
return ret
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class RetryingSession(CertifiSession):
|
|
|
|
|
class RetryingSession(CFSession):
|
|
|
|
|
proxied_functions = ("get", "post")
|
|
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
|
super(RetryingSession, self).__init__()
|
|
|
|
|
self.verify = pem_file
|
|
|
|
|
|
|
|
|
|
proxy = os.environ.get('SZ_HTTP_PROXY')
|
|
|
|
|
if proxy:
|
|
|
|
|