From 4c3967fd6d8d3f4314076bc686ce9466f1dfc7a6 Mon Sep 17 00:00:00 2001 From: Tejasvi Nareddy Date: Sun, 6 Jan 2019 22:33:36 -0500 Subject: [PATCH 1/2] feat: expose response times - Add hook to FuturesSession to calculate absolute response time (ms) - In verbose mode, display response times for each request - Always outputs response times to CSV --- sherlock.py | 122 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 47 deletions(-) diff --git a/sherlock.py b/sherlock.py index 304b5ca0..4f2b058c 100644 --- a/sherlock.py +++ b/sherlock.py @@ -13,11 +13,12 @@ import os import sys import platform import re -from argparse import ArgumentParser, RawDescriptionHelpFormatter -from concurrent.futures import ThreadPoolExecutor +from time import time import requests -from colorama import Back, Fore, Style, init +from argparse import ArgumentParser, RawDescriptionHelpFormatter +from concurrent.futures import ThreadPoolExecutor +from colorama import Fore, Style, init from requests_futures.sessions import FuturesSession from torrequest import TorRequest @@ -28,28 +29,75 @@ amount=0 # TODO: fix tumblr +# Extends FutureSession to add response time metric +# This is taken (almost) directly from here: https://github.com/ross/requests-futures#working-in-the-background +class ElapsedFuturesSession(FuturesSession): + + def request(self, method, url, hooks={}, *args, **kwargs): + start = time() + + def timing(r, *args, **kwargs): + elapsed_sec = time() - start + r.elapsed = round(elapsed_sec * 1000) + + try: + if isinstance(hooks['response'], (list, tuple)): + # needs to be first so we don't time other hooks execution + hooks['response'].insert(0, timing) + else: + hooks['response'] = [timing, hooks['response']] + except KeyError: + hooks['response'] = timing + + return super(ElapsedFuturesSession, self).request(method, url, hooks=hooks, *args, **kwargs) + + def open_file(fname): return open(fname, "a") + def write_to_file(url, f): f.write(url + "\n") + def final_score(amount, f): f.write("Total: "+str(amount) + "\n") -def print_error(err, errstr, var, debug=False): + +def print_error(err, errstr, var, verbose=False): print(Style.BRIGHT + Fore.WHITE + "[" + Fore.RED + "-" + Fore.WHITE + "]" + Fore.RED + f" {errstr}" + - Fore.YELLOW + f" {err if debug else var}") + Fore.YELLOW + f" {err if verbose else var}") + + +def create_response_time(response_time, verbose): + return " [{} ms]".format(response_time) if verbose else "" + + +def print_found(social_network, url, response_time, verbose=False): + print((Style.BRIGHT + Fore.WHITE + "[" + + Fore.GREEN + "+" + + Fore.WHITE + "]" + + create_response_time(response_time, verbose) + + Fore.GREEN + " {}:").format(social_network), url) + + +def print_not_found(social_network, response_time, verbose=False): + print((Style.BRIGHT + Fore.WHITE + "[" + + Fore.RED + "-" + + Fore.WHITE + "]" + + create_response_time(response_time, verbose) + + Fore.GREEN + " {}:" + + Fore.YELLOW + " Not Found!").format(social_network)) def get_response(request_future, error_type, social_network, verbose=False): try: rsp = request_future.result() if rsp.status_code: - return rsp, error_type + return rsp, error_type, rsp.elapsed except requests.exceptions.HTTPError as errh: print_error(errh, "HTTP Error:", social_network, verbose) except requests.exceptions.ConnectionError as errc: @@ -58,7 +106,7 @@ def get_response(request_future, error_type, social_network, verbose=False): print_error(errt, "Timeout Error:", social_network, verbose) except requests.exceptions.RequestException as err: print_error(err, "Unknown error:", social_network, verbose) - return None, "" + return None, "", -1 def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False): @@ -119,8 +167,8 @@ def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False): underlying_request = TorRequest() underlying_session = underlying_request.session() - # Create multi-threaded session for all requests - session = FuturesSession(executor=executor, session=underlying_session) + # Create multi-threaded session for all requests. Use our custom FuturesSession that exposes response time + session = ElapsedFuturesSession(executor=executor, session=underlying_session) # Results from analysis of all sites results_total = {} @@ -193,10 +241,10 @@ def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False): # Retrieve future and ensure it has finished future = net_info["request_future"] - r, error_type = get_response(request_future=future, - error_type=error_type, - social_network=social_network, - verbose=verbose) + r, error_type, response_time = get_response(request_future=future, + error_type=error_type, + social_network=social_network, + verbose=verbose) # Attempt to get request information try: @@ -212,59 +260,35 @@ def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False): error = net_info.get("errorMsg") # Checks if the error message is in the HTML if not error in r.text: - - print((Style.BRIGHT + Fore.WHITE + "[" + - Fore.GREEN + "+" + - Fore.WHITE + "]" + - Fore.GREEN + " {}:").format(social_network), url) + print_found(social_network, url, response_time, verbose) write_to_file(url, f) exists = "yes" amount=amount+1 else: - print((Style.BRIGHT + Fore.WHITE + "[" + - Fore.RED + "-" + - Fore.WHITE + "]" + - Fore.GREEN + " {}:" + - Fore.YELLOW + " Not Found!").format(social_network)) + print_not_found(social_network, response_time, verbose) exists = "no" elif error_type == "status_code": # Checks if the status code of the response is 2XX if not r.status_code >= 300 or r.status_code < 200: - - print((Style.BRIGHT + Fore.WHITE + "[" + - Fore.GREEN + "+" + - Fore.WHITE + "]" + - Fore.GREEN + " {}:").format(social_network), url) + print_found(social_network, url, response_time, verbose) write_to_file(url, f) exists = "yes" amount=amount+1 else: - print((Style.BRIGHT + Fore.WHITE + "[" + - Fore.RED + "-" + - Fore.WHITE + "]" + - Fore.GREEN + " {}:" + - Fore.YELLOW + " Not Found!").format(social_network)) + print_not_found(social_network, response_time, verbose) exists = "no" elif error_type == "response_url": error = net_info.get("errorUrl") # Checks if the redirect url is the same as the one defined in data.json if not error in r.url: - - print((Style.BRIGHT + Fore.WHITE + "[" + - Fore.GREEN + "+" + - Fore.WHITE + "]" + - Fore.GREEN + " {}:").format(social_network), url) + print_found(social_network, url, response_time, verbose) write_to_file(url, f) exists = "yes" amount=amount+1 else: - print((Style.BRIGHT + Fore.WHITE + "[" + - Fore.RED + "-" + - Fore.WHITE + "]" + - Fore.GREEN + " {}:" + - Fore.YELLOW + " Not Found!").format(social_network)) + print_not_found(social_network, response_time, verbose) exists = "no" elif error_type == "": @@ -276,11 +300,12 @@ def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False): exists = "error" # Save exists flag - results_site['exists'] = exists + results_site['exists'] = exists # Save results from request - results_site['http_status'] = http_status + results_site['http_status'] = http_status results_site['response_text'] = response_text + results_site['response_time_ms'] = response_time # Add this site's results into final dictionary with all of the other results. results_total[social_network] = results_site @@ -393,7 +418,8 @@ def main(): 'url_main', 'url_user', 'exists', - 'http_status' + 'http_status', + 'response_time_ms' ] ) for site in results: @@ -402,9 +428,11 @@ def main(): results[site]['url_main'], results[site]['url_user'], results[site]['exists'], - results[site]['http_status'] + results[site]['http_status'], + results[site]['response_time_ms'] ] ) + if __name__ == "__main__": main() \ No newline at end of file From 038d5314831ae420c2fe0fa2dd3c1540677c36c8 Mon Sep 17 00:00:00 2001 From: Tejasvi Nareddy Date: Mon, 7 Jan 2019 12:04:55 -0500 Subject: [PATCH 2/2] doc: response time - Document new feature for response times - Refactor --- README.md | 2 +- sherlock.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 6b964136..94aa6391 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ optional arguments: -h, --help show this help message and exit --version Display version information and dependencies. --verbose, -v, -d, --debug - Display extra debugging information. + Display extra debugging information and metrics. --quiet, -q Disable debugging information (Default Option). --tor, -t Make requests over TOR; increases runtime; requires TOR to be installed and in system path. diff --git a/sherlock.py b/sherlock.py index 4f2b058c..e61ec4a0 100644 --- a/sherlock.py +++ b/sherlock.py @@ -29,9 +29,12 @@ amount=0 # TODO: fix tumblr -# Extends FutureSession to add response time metric -# This is taken (almost) directly from here: https://github.com/ross/requests-futures#working-in-the-background class ElapsedFuturesSession(FuturesSession): + """ + Extends FutureSession to add a response time metric to each request. + + This is taken (almost) directly from here: https://github.com/ross/requests-futures#working-in-the-background + """ def request(self, method, url, hooks={}, *args, **kwargs): start = time() @@ -72,7 +75,7 @@ def print_error(err, errstr, var, verbose=False): Fore.YELLOW + f" {err if verbose else var}") -def create_response_time(response_time, verbose): +def format_response_time(response_time, verbose): return " [{} ms]".format(response_time) if verbose else "" @@ -80,7 +83,7 @@ def print_found(social_network, url, response_time, verbose=False): print((Style.BRIGHT + Fore.WHITE + "[" + Fore.GREEN + "+" + Fore.WHITE + "]" + - create_response_time(response_time, verbose) + + format_response_time(response_time, verbose) + Fore.GREEN + " {}:").format(social_network), url) @@ -88,7 +91,7 @@ def print_not_found(social_network, response_time, verbose=False): print((Style.BRIGHT + Fore.WHITE + "[" + Fore.RED + "-" + Fore.WHITE + "]" + - create_response_time(response_time, verbose) + + format_response_time(response_time, verbose) + Fore.GREEN + " {}:" + Fore.YELLOW + " Not Found!").format(social_network)) @@ -336,7 +339,7 @@ def main(): ) parser.add_argument("--verbose", "-v", "-d", "--debug", action="store_true", dest="verbose", default=False, - help="Display extra debugging information." + help="Display extra debugging information and metrics." ) parser.add_argument("--quiet", "-q", action="store_false", dest="verbose",