open-source-mirrors
/
sherlock
mirror of https://github.com/sherlock-project/sherlock
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1246 from benni347/master

Browse Source 
Refractored sites.py, sherlock.py and notify.py.
pull/1283/head^2
Yahya SayadArbabi 3 years ago committed by GitHub
parent 5a02bae856 cd26a0df41
commit 5d8d751796
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 80 additions and 86 deletions
Show Stats Download Patch File Download Diff File

26
sherlock/notify.py
Unescape View File

@ -33,7 +33,7 @@ class QueryNotify:
self.result = result
return
# return
def start(self, message=None):
"""Notify Start.
@ -52,7 +52,7 @@ class QueryNotify:
Nothing.
"""
return
# return
def update(self, result):
"""Notify Update.
@ -71,7 +71,7 @@ class QueryNotify:
self.result = result
return
# return
def finish(self, message=None):
"""Notify Finish.
@ -90,7 +90,7 @@ class QueryNotify:
Nothing.
"""
return
# return
def __str__(self):
"""Convert Object To String.
@ -101,9 +101,7 @@ class QueryNotify:
Return Value:
Nicely formatted string to get information about this object.
"""
result = str(self.result)
return result
return str(self.result)
class QueryNotifyPrint(QueryNotify):
@ -150,6 +148,7 @@ class QueryNotifyPrint(QueryNotify):
"""
title = "Checking username"
print(Style.BRIGHT + Fore.GREEN + "[" +
Fore.YELLOW + "*" +
Fore.GREEN + f"] {title}" +
@ -184,7 +183,7 @@ class QueryNotifyPrint(QueryNotify):
# An empty line between first line and the result(more clear output)
return
# return
def update(self, result):
"""Notify Update.
@ -201,7 +200,7 @@ class QueryNotifyPrint(QueryNotify):
"""
self.result = result
if self.verbose == False or self.result.query_time is None:
if self.verbose is False or self.result.query_time is None:
response_time_text = ""
else:
response_time_text = f" [{round(self.result.query_time * 1000)} ms]"
@ -246,8 +245,9 @@ class QueryNotifyPrint(QueryNotify):
else:
# It should be impossible to ever get here...
raise ValueError(f"Unknown Query Status '{str(result.status)}' for "
f"site '{self.result.site_name}'")
raise ValueError(
f"Unknown Query Status '{result.status}' for site '{self.result.site_name}'"
)
return
@ -260,6 +260,4 @@ class QueryNotifyPrint(QueryNotify):
Return Value:
Nicely formatted string to get information about this object.
"""
result = str(self.result)
return result
return str(self.result)

47
sherlock/sherlock.py
Unescape View File

@ -30,7 +30,7 @@ __version__ = "0.14.0"
class SherlockFuturesSession(FuturesSession):
def request(self, method, url, hooks={}, *args, **kwargs):
def request(self, method, url, hooks=None, *args, **kwargs):
"""Request URL.
This extends the FuturesSession request method to calculate a response
@ -52,6 +52,8 @@ class SherlockFuturesSession(FuturesSession):
Request object.
"""
# Record the start time for the request.
if hooks is None:
hooks = {}
start = monotonic()
def response_time(resp, *args, **kwargs):
@ -94,12 +96,11 @@ class SherlockFuturesSession(FuturesSession):
def get_response(request_future, error_type, social_network):
# Default for Response object if some failure occurs.
response = None
error_context = "General Unknown Error"
expection_text = None
exception_text = None
try:
response = request_future.result()
if response.status_code:
@ -107,21 +108,21 @@ def get_response(request_future, error_type, social_network):
error_context = None
except requests.exceptions.HTTPError as errh:
error_context = "HTTP Error"
expection_text = str(errh)
exception_text = str(errh)
except requests.exceptions.ProxyError as errp:
error_context = "Proxy Error"
expection_text = str(errp)
exception_text = str(errp)
except requests.exceptions.ConnectionError as errc:
error_context = "Error Connecting"
expection_text = str(errc)
exception_text = str(errc)
except requests.exceptions.Timeout as errt:
error_context = "Timeout Error"
expection_text = str(errt)
exception_text = str(errt)
except requests.exceptions.RequestException as err:
error_context = "Unknown Error"
expection_text = str(err)
exception_text = str(err)
return response, error_context, expection_text
return response, error_context, exception_text
def interpolate_string(object, username):
@ -204,10 +205,9 @@ def sherlock(username, site_data, query_notify,
for social_network, net_info in site_data.items():
# Results from analysis of this specific site
results_site = {}
results_site = {"url_main": net_info.get("urlMain")}
# Record URL of main site
results_site["url_main"] = net_info.get("urlMain")
# A user agent is needed because some sites don't return the correct
# information since they think that we are bots (Which we actually are...)
@ -225,7 +225,7 @@ def sherlock(username, site_data, query_notify,
# Don't make request if username is invalid for the site
regex_check = net_info.get("regexCheck")
if regex_check and re.search(regex_check, username) is None:
# No need to do the check at the site: this user name is not allowed.
# No need to do the check at the site: this username is not allowed.
results_site["status"] = QueryResult(username,
social_network,
url,
@ -310,7 +310,7 @@ def sherlock(username, site_data, query_notify,
if unique_tor:
underlying_request.reset_identity()
# Add this site's results into final dictionary with all of the other results.
# Add this site's results into final dictionary with all the other results.
results_total[social_network] = results_site
# Open the file containing account links
@ -333,7 +333,7 @@ def sherlock(username, site_data, query_notify,
# Retrieve future and ensure it has finished
future = net_info["request_future"]
r, error_text, expection_text = get_response(request_future=future,
r, error_text, exception_text = get_response(request_future=future,
error_type=error_type,
social_network=social_network)
@ -367,7 +367,7 @@ def sherlock(username, site_data, query_notify,
errors = net_info.get("errorMsg")
# errors will hold the error message
# it can be string or list
# by insinstance method we can detect that
# by isinstance method we can detect that
# and handle the case for strings as normal procedure
# and if its list we can iterate the errors
if isinstance(errors, str):
@ -483,8 +483,7 @@ def timeout_check(value):
def main():
version_string = f"%(prog)s {__version__}\n" + \
version_string = f"%(prog)s {__version__}\n" + \
f"{requests.__description__}: {requests.__version__}\n" + \
f"Python: {platform.python_version()}"
@ -492,11 +491,11 @@ def main():
description=f"{module_name} (Version {__version__})"
)
parser.add_argument("--version",
action="version", version=version_string,
action="version", version=version_string,
help="Display version information and dependencies."
)
parser.add_argument("--verbose", "-v", "-d", "--debug",
action="store_true", dest="verbose", default=False,
action="store_true", dest="verbose", default=False,
help="Display extra debugging information and metrics."
)
parser.add_argument("--folderoutput", "-fo", dest="folderoutput",
@ -512,7 +511,7 @@ def main():
action="store_true", dest="unique_tor", default=False,
help="Make requests over Tor with new Tor circuit after each request; increases runtime; requires Tor to be installed and in system path.")
parser.add_argument("--csv",
action="store_true", dest="csv", default=False,
action="store_true", dest="csv", default=False,
help="Create Comma-Separated Values (CSV) File."
)
parser.add_argument("--site",
@ -588,7 +587,8 @@ def main():
if args.tor or args.unique_tor:
print("Using Tor to make requests")
print("Warning: some websites might refuse connecting over Tor, so note that using this option might increase connection errors.")
print(
"Warning: some websites might refuse connecting over Tor, so note that using this option might increase connection errors.")
if args.no_color:
# Disable color output.
@ -621,10 +621,7 @@ def main():
# Create original dictionary from SitesInformation() object.
# Eventually, the rest of the code will be updated to use the new object
# directly, but this will glue the two pieces together.
site_data_all = {}
for site in sites:
site_data_all[site.name] = site.information
site_data_all = {site.name: site.information for site in sites}
if args.site_list is None:
# Not desired to look at a sub-set of sites
site_data = site_data_all

90
sherlock/sites.py
Unescape View File

@ -1,21 +1,18 @@
"""Sherlock Sites Information Module
This module supports storing information about web sites.
This module supports storing information about websites.
This is the raw data that will be used to search for usernames.
"""
import os
import json
import operator
import requests
import sys
class SiteInformation():
class SiteInformation:
def __init__(self, name, url_home, url_username_format, username_claimed,
username_unclaimed, information):
"""Create Site Information Object.
Contains information about a specific web site.
Contains information about a specific website.
Keyword Arguments:
self -- This object.
@ -30,13 +27,13 @@ class SiteInformation():
indicates that the individual
usernames would show up under the
"https://somesite.com/users/" area of
the web site.
the website.
username_claimed -- String containing username which is known
to be claimed on web site.
to be claimed on website.
username_unclaimed -- String containing username which is known
to be unclaimed on web site.
to be unclaimed on website.
information -- Dictionary containing all known information
about web site.
about website.
NOTE: Custom information about how to
actually detect the existence of the
username will be included in this
@ -49,13 +46,13 @@ class SiteInformation():
Nothing.
"""
self.name = name
self.url_home = url_home
self.name = name
self.url_home = url_home
self.url_username_format = url_username_format
self.username_claimed = username_claimed
self.username_unclaimed = username_unclaimed
self.information = information
self.username_claimed = username_claimed
self.username_unclaimed = username_unclaimed
self.information = information
return
@ -72,11 +69,11 @@ class SiteInformation():
return f"{self.name} ({self.url_home})"
class SitesInformation():
class SitesInformation:
def __init__(self, data_file_path=None):
"""Create Sites Information Object.
Contains information about all supported web sites.
Contains information about all supported websites.
Keyword Arguments:
self -- This object.
@ -109,7 +106,7 @@ class SitesInformation():
if data_file_path is None:
# The default data file is the live data.json which is in the GitHub repo. The reason why we are using
# this instead of the local one is so that the user has the most up to date data. This prevents
# this instead of the local one is so that the user has the most up-to-date data. This prevents
# users from creating issue about false positives which has already been fixed or having outdated data
data_file_path = "https://raw.githubusercontent.com/sherlock-project/sherlock/master/sherlock/resources/data.json"
@ -117,26 +114,29 @@ class SitesInformation():
if not data_file_path.lower().endswith(".json"):
raise FileNotFoundError(f"Incorrect JSON file extension for data file '{data_file_path}'.")
if "http://" == data_file_path[:7].lower() or "https://" == data_file_path[:8].lower():
if (
data_file_path[:7].lower() == "http://"
or data_file_path[:8].lower() == "https://"
):
# Reference is to a URL.
try:
response = requests.get(url=data_file_path)
except Exception as error:
raise FileNotFoundError(f"Problem while attempting to access "
f"data file URL '{data_file_path}': "
f"{str(error)}"
)
if response.status_code == 200:
try:
site_data = response.json()
except Exception as error:
raise ValueError(f"Problem parsing json contents at "
f"'{data_file_path}': {str(error)}."
)
else:
raise FileNotFoundError(
f"Problem while attempting to access data file URL '{data_file_path}': {error}"
)
if response.status_code != 200:
raise FileNotFoundError(f"Bad response while accessing "
f"data file URL '{data_file_path}'."
)
)
try:
site_data = response.json()
except Exception as error:
raise ValueError(
f"Problem parsing json contents at '{data_file_path}': {error}."
)
else:
# Reference is to a file.
try:
@ -144,17 +144,18 @@ class SitesInformation():
try:
site_data = json.load(file)
except Exception as error:
raise ValueError(f"Problem parsing json contents at "
f"'{data_file_path}': {str(error)}."
)
except FileNotFoundError as error:
raise ValueError(
f"Problem parsing json contents at '{data_file_path}': {error}."
)
except FileNotFoundError:
raise FileNotFoundError(f"Problem while attempting to access "
f"data file '{data_file_path}'."
)
)
self.sites = {}
# Add all of site information from the json file to internal site list.
# Add all site information from the json file to internal site list.
for site_name in site_data:
try:
@ -165,12 +166,11 @@ class SitesInformation():
site_data[site_name]["username_claimed"],
site_data[site_name]["username_unclaimed"],
site_data[site_name]
)
)
except KeyError as error:
raise ValueError(f"Problem parsing json contents at "
f"'{data_file_path}': "
f"Missing attribute {str(error)}."
)
raise ValueError(
f"Problem parsing json contents at '{data_file_path}': Missing attribute {error}."
)
return
@ -184,9 +184,7 @@ class SitesInformation():
List of strings containing names of sites.
"""
site_names = sorted([site.name for site in self], key=str.lower)
return site_names
return sorted([site.name for site in self], key=str.lower)
def __iter__(self):
"""Iterator For Object.

3
site_list.py
Unescape View File

@ -3,9 +3,10 @@ This module generates the listing of supported sites
which can be found in sites.md
It also organizes all the sites in alphanumeric order
"""
import json
pool = list()
pool = []
with open("sherlock/resources/data.json", "r", encoding="utf-8") as data_file:
data = json.load(data_file)

Write Preview
Loading…
Cancel
Save