open-source-mirrors
/
sherlock
mirror of https://github.com/sherlock-project/sherlock
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #109 from nareddyt/response-time-metrics

Browse Source 
Response time metrics
pull/114/head
Tejasvi (Teju) Nareddy 6 years ago committed by GitHub
parent 2dd5a5e145 038d531483
commit d0ecb70d84
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 80 additions and 49 deletions
Show Stats Download Patch File Download Diff File

2
README.md
Unescape View File

@ -37,7 +37,7 @@ optional arguments:
-h, --help show this help message and exit
--version Display version information and dependencies.
--verbose, -v, -d, --debug
Display extra debugging information.
Display extra debugging information and metrics.
--quiet, -q Disable debugging information (Default Option).
--tor, -t Make requests over TOR; increases runtime; requires
TOR to be installed and in system path.

127
sherlock.py
Unescape View File

@ -13,11 +13,12 @@ import os
import sys
import platform
import re
from argparse import ArgumentParser, RawDescriptionHelpFormatter
from concurrent.futures import ThreadPoolExecutor
from time import time
import requests
from colorama import Back, Fore, Style, init
from argparse import ArgumentParser, RawDescriptionHelpFormatter
from concurrent.futures import ThreadPoolExecutor
from colorama import Fore, Style, init
from requests_futures.sessions import FuturesSession
from torrequest import TorRequest
@ -28,28 +29,78 @@ amount=0
# TODO: fix tumblr
class ElapsedFuturesSession(FuturesSession):
"""
Extends FutureSession to add a response time metric to each request.
This is taken (almost) directly from here: https://github.com/ross/requests-futures#working-in-the-background
"""
def request(self, method, url, hooks={}, *args, **kwargs):
start = time()
def timing(r, *args, **kwargs):
elapsed_sec = time() - start
r.elapsed = round(elapsed_sec * 1000)
try:
if isinstance(hooks['response'], (list, tuple)):
# needs to be first so we don't time other hooks execution
hooks['response'].insert(0, timing)
else:
hooks['response'] = [timing, hooks['response']]
except KeyError:
hooks['response'] = timing
return super(ElapsedFuturesSession, self).request(method, url, hooks=hooks, *args, **kwargs)
def open_file(fname):
return open(fname, "a")
def write_to_file(url, f):
f.write(url + "\n")
def final_score(amount, f):
f.write("Total: "+str(amount) + "\n")
def print_error(err, errstr, var, debug=False):
def print_error(err, errstr, var, verbose=False):
print(Style.BRIGHT + Fore.WHITE + "[" +
Fore.RED + "-" +
Fore.WHITE + "]" +
Fore.RED + f" {errstr}" +
Fore.YELLOW + f" {err if debug else var}")
Fore.YELLOW + f" {err if verbose else var}")
def format_response_time(response_time, verbose):
return " [{} ms]".format(response_time) if verbose else ""
def print_found(social_network, url, response_time, verbose=False):
print((Style.BRIGHT + Fore.WHITE + "[" +
Fore.GREEN + "+" +
Fore.WHITE + "]" +
format_response_time(response_time, verbose) +
Fore.GREEN + " {}:").format(social_network), url)
def print_not_found(social_network, response_time, verbose=False):
print((Style.BRIGHT + Fore.WHITE + "[" +
Fore.RED + "-" +
Fore.WHITE + "]" +
format_response_time(response_time, verbose) +
Fore.GREEN + " {}:" +
Fore.YELLOW + " Not Found!").format(social_network))
def get_response(request_future, error_type, social_network, verbose=False):
try:
rsp = request_future.result()
if rsp.status_code:
return rsp, error_type
return rsp, error_type, rsp.elapsed
except requests.exceptions.HTTPError as errh:
print_error(errh, "HTTP Error:", social_network, verbose)
except requests.exceptions.ConnectionError as errc:
@ -58,7 +109,7 @@ def get_response(request_future, error_type, social_network, verbose=False):
print_error(errt, "Timeout Error:", social_network, verbose)
except requests.exceptions.RequestException as err:
print_error(err, "Unknown error:", social_network, verbose)
return None, ""
return None, "", -1
def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False):
@ -119,8 +170,8 @@ def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False):
underlying_request = TorRequest()
underlying_session = underlying_request.session()
# Create multi-threaded session for all requests
session = FuturesSession(executor=executor, session=underlying_session)
# Create multi-threaded session for all requests. Use our custom FuturesSession that exposes response time
session = ElapsedFuturesSession(executor=executor, session=underlying_session)
# Results from analysis of all sites
results_total = {}
@ -193,10 +244,10 @@ def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False):
# Retrieve future and ensure it has finished
future = net_info["request_future"]
r, error_type = get_response(request_future=future,
error_type=error_type,
social_network=social_network,
verbose=verbose)
r, error_type, response_time = get_response(request_future=future,
error_type=error_type,
social_network=social_network,
verbose=verbose)
# Attempt to get request information
try:
@ -212,59 +263,35 @@ def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False):
error = net_info.get("errorMsg")
# Checks if the error message is in the HTML
if not error in r.text:
print((Style.BRIGHT + Fore.WHITE + "[" +
Fore.GREEN + "+" +
Fore.WHITE + "]" +
Fore.GREEN + " {}:").format(social_network), url)
print_found(social_network, url, response_time, verbose)
write_to_file(url, f)
exists = "yes"
amount=amount+1
else:
print((Style.BRIGHT + Fore.WHITE + "[" +
Fore.RED + "-" +
Fore.WHITE + "]" +
Fore.GREEN + " {}:" +
Fore.YELLOW + " Not Found!").format(social_network))
print_not_found(social_network, response_time, verbose)
exists = "no"
elif error_type == "status_code":
# Checks if the status code of the response is 2XX
if not r.status_code >= 300 or r.status_code < 200:
print((Style.BRIGHT + Fore.WHITE + "[" +
Fore.GREEN + "+" +
Fore.WHITE + "]" +
Fore.GREEN + " {}:").format(social_network), url)
print_found(social_network, url, response_time, verbose)
write_to_file(url, f)
exists = "yes"
amount=amount+1
else:
print((Style.BRIGHT + Fore.WHITE + "[" +
Fore.RED + "-" +
Fore.WHITE + "]" +
Fore.GREEN + " {}:" +
Fore.YELLOW + " Not Found!").format(social_network))
print_not_found(social_network, response_time, verbose)
exists = "no"
elif error_type == "response_url":
error = net_info.get("errorUrl")
# Checks if the redirect url is the same as the one defined in data.json
if not error in r.url:
print((Style.BRIGHT + Fore.WHITE + "[" +
Fore.GREEN + "+" +
Fore.WHITE + "]" +
Fore.GREEN + " {}:").format(social_network), url)
print_found(social_network, url, response_time, verbose)
write_to_file(url, f)
exists = "yes"
amount=amount+1
else:
print((Style.BRIGHT + Fore.WHITE + "[" +
Fore.RED + "-" +
Fore.WHITE + "]" +
Fore.GREEN + " {}:" +
Fore.YELLOW + " Not Found!").format(social_network))
print_not_found(social_network, response_time, verbose)
exists = "no"
elif error_type == "":
@ -276,11 +303,12 @@ def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False):
exists = "error"
# Save exists flag
results_site['exists'] = exists
results_site['exists'] = exists
# Save results from request
results_site['http_status'] = http_status
results_site['http_status'] = http_status
results_site['response_text'] = response_text
results_site['response_time_ms'] = response_time
# Add this site's results into final dictionary with all of the other results.
results_total[social_network] = results_site
@ -311,7 +339,7 @@ def main():
)
parser.add_argument("--verbose", "-v", "-d", "--debug",
action="store_true", dest="verbose", default=False,
help="Display extra debugging information."
help="Display extra debugging information and metrics."
)
parser.add_argument("--quiet", "-q",
action="store_false", dest="verbose",
@ -393,7 +421,8 @@ def main():
'url_main',
'url_user',
'exists',
'http_status'
'http_status',
'response_time_ms'
]
)
for site in results:
@ -402,9 +431,11 @@ def main():
results[site]['url_main'],
results[site]['url_user'],
results[site]['exists'],
results[site]['http_status']
results[site]['http_status'],
results[site]['response_time_ms']
]
)
if __name__ == "__main__":
main()
Write Preview
Loading…
Cancel
Save