From b8737bd75f7d36cb02caa473ad6a90732306edfd Mon Sep 17 00:00:00 2001 From: theodosisathanasakis <32517114+theodosisathanasakis@users.noreply.github.com> Date: Thu, 3 Jan 2019 03:25:27 +0200 Subject: [PATCH 01/14] Add "regexCheck" for Facebook and Flipboard Facebook from: https://www.facebook.com/help/409473442437047 but I don't know how to handle this requirement: "They can't contain generic terms or extensions (.com, .net)". Flipboard from: https://flipboard.helpshift.com/a/flipboard/?s=managing-your-account&f=help-with-username-issues&p=web --- data.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/data.json b/data.json index 8228645d..0ff11370 100644 --- a/data.json +++ b/data.json @@ -14,7 +14,8 @@ "Facebook": { "url": "https://www.facebook.com/{}", "urlMain": "https://www.facebook.com/", - "errorType": "status_code" + "errorType": "status_code", + "regexCheck": "^([a-zA-Z0-9.]){5,}$" }, "YouTube": { "url": "https://www.youtube.com/{}", @@ -108,7 +109,8 @@ "url": "https://flipboard.com/@{}", "urlMain": "https://flipboard.com/", "errorType": "message", - "errorMsg": "loading" + "errorMsg": "loading", + "regexCheck": "^([a-zA-Z0-9_]){1,15}$" }, "SlideShare": { "url": "https://slideshare.net/{}", From df1f35bc88a2d68ce57c22bfd840d7c056f43eba Mon Sep 17 00:00:00 2001 From: Jen-Chieh Date: Fri, 4 Jan 2019 13:17:23 +0800 Subject: [PATCH 02/14] Use Colorama for better readability. --- sherlock.py | 89 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 23 deletions(-) diff --git a/sherlock.py b/sherlock.py index 93ea0296..0d976395 100644 --- a/sherlock.py +++ b/sherlock.py @@ -36,7 +36,11 @@ def final_score(amount, fname): f.write("Total: "+str(amount) + "\n") def print_error(err, errstr, var, debug=False): - print(f"\033[37;1m[\033[91;1m-\033[37;1m]\033[91;1m {errstr}\033[93;1m {err if debug else var}") + print(Style.BRIGHT + Fore.WHITE + "[" + + Fore.RED + "-" + + Fore.WHITE + "]" + + Fore.RED + f" {errstr}" + + Fore.YELLOW + f" {err if debug else var}") def get_response(request_future, error_type, social_network, verbose=False): @@ -84,9 +88,16 @@ def sherlock(username, verbose=False, tor=False, unique_tor=False): if os.path.isfile(fname): os.remove(fname) - print("\033[1;92m[\033[0m\033[1;77m*\033[0m\033[1;92m] Removing previous file:\033[1;37m {}\033[0m".format(fname)) + print((Style.BRIGHT + Fore.GREEN + "[" + + Fore.YELLOW + "*" + + Fore.GREEN + "] Removing previous file:" + + Fore.WHITE + " {}").format(fname)) - print("\033[1;92m[\033[0m\033[1;77m*\033[0m\033[1;92m] Checking username\033[0m\033[1;37m {}\033[0m\033[1;92m on: \033[0m".format(username)) + print((Style.BRIGHT + Fore.GREEN + "[" + + Fore.YELLOW + "*" + + Fore.GREEN + "] Checking username" + + Fore.WHITE + " {}" + + Fore.GREEN + " on:").format(username)) # A user agent is needed because some sites don't # return the correct information since they think that @@ -129,7 +140,11 @@ def sherlock(username, verbose=False, tor=False, unique_tor=False): regex_check = net_info.get("regexCheck") if regex_check and re.search(regex_check, username) is None: # No need to do the check at the site: this user name is not allowed. - print("\033[37;1m[\033[91;1m-\033[37;1m]\033[92;1m {}:\033[93;1m Illegal Username Format For This Site!".format(social_network)) + print((Style.BRIGHT + Fore.WHITE + "[" + + Fore.RED + "-" + + Fore.WHITE + "]" + + Fore.GREEN + " {}:" + + Fore.YELLOW + " Illegal Username Format For This Site!").format(social_network)) results_site["exists"] = "illegal" else: # URL of user on site (if it exists) @@ -197,24 +212,38 @@ def sherlock(username, verbose=False, tor=False, unique_tor=False): # Checks if the error message is in the HTML if not error in r.text: - print("\033[37;1m[\033[92;1m+\033[37;1m]\033[92;1m {}:\033[0m".format(social_network), url) + print((Style.BRIGHT + Fore.WHITE + "[" + + Fore.GREEN + "+" + + Fore.WHITE + "]" + + Fore.GREEN + " {}:").format(social_network), url) write_to_file(url, fname) exists = "yes" amount=amount+1 else: - print("\033[37;1m[\033[91;1m-\033[37;1m]\033[92;1m {}:\033[93;1m Not Found!".format(social_network)) + print((Style.BRIGHT + Fore.WHITE + "[" + + Fore.RED + "-" + + Fore.WHITE + "]" + + Fore.GREEN + " {}:" + + Fore.YELLOW + " Not Found!").format(social_network)) exists = "no" elif error_type == "status_code": # Checks if the status code of the response is 404 if not r.status_code == 404: - print("\033[37;1m[\033[92;1m+\033[37;1m]\033[92;1m {}:\033[0m".format(social_network), url) + print((Style.BRIGHT + Fore.WHITE + "[" + + Fore.GREEN + "+" + + Fore.WHITE + "]" + + Fore.GREEN + " {}:").format(social_network), url) write_to_file(url, fname) exists = "yes" amount=amount+1 else: - print("\033[37;1m[\033[91;1m-\033[37;1m]\033[92;1m {}:\033[93;1m Not Found!".format(social_network)) + print((Style.BRIGHT + Fore.WHITE + "[" + + Fore.RED + "-" + + Fore.WHITE + "]" + + Fore.GREEN + " {}:" + + Fore.YELLOW + " Not Found!").format(social_network)) exists = "no" elif error_type == "response_url": @@ -222,16 +251,27 @@ def sherlock(username, verbose=False, tor=False, unique_tor=False): # Checks if the redirect url is the same as the one defined in data.json if not error in r.url: - print("\033[37;1m[\033[92;1m+\033[37;1m]\033[92;1m {}:\033[0m".format(social_network), url) + print((Style.BRIGHT + Fore.WHITE + "[" + + Fore.GREEN + "+" + + Fore.WHITE + "]" + + Fore.GREEN + " {}:").format(social_network), url) write_to_file(url, fname) exists = "yes" amount=amount+1 else: - print("\033[37;1m[\033[91;1m-\033[37;1m]\033[92;1m {}:\033[93;1m Not Found!".format(social_network)) + print((Style.BRIGHT + Fore.WHITE + "[" + + Fore.RED + "-" + + Fore.WHITE + "]" + + Fore.GREEN + " {}:" + + Fore.YELLOW + " Not Found!").format(social_network)) exists = "no" elif error_type == "": - print("\033[37;1m[\033[91;1m-\033[37;1m]\033[92;1m {}:\033[93;1m Error!".format(social_network)) + print((Style.BRIGHT + Fore.WHITE + "[" + + Fore.RED + "-" + + Fore.WHITE + "]" + + Fore.GREEN + " {}:" + + Fore.YELLOW + " Error!").format(social_network)) exists = "error" # Save exists flag @@ -244,7 +284,10 @@ def sherlock(username, verbose=False, tor=False, unique_tor=False): # Add this site's results into final dictionary with all of the other results. results_total[social_network] = results_site - print("\033[1;92m[\033[0m\033[1;77m*\033[0m\033[1;92m] Saved: \033[37;1m{}\033[0m".format(username+".txt")) + print((Style.BRIGHT + Fore.GREEN + "[" + + Fore.YELLOW + "*" + + Fore.GREEN + "] Saved: " + + Fore.WHITE + "{}").format(username+".txt")) final_score(amount, fname) return results_total @@ -252,7 +295,7 @@ def sherlock(username, verbose=False, tor=False, unique_tor=False): def main(): # Colorama module's initialization. - init() + init(autoreset=True) version_string = f"%(prog)s {__version__}\n" + \ f"{requests.__description__}: {requests.__version__}\n" + \ @@ -292,16 +335,16 @@ def main(): args = parser.parse_args() # Banner - print( -"""\033[37;1m .\"\"\"-. -\033[37;1m / \\ -\033[37;1m ____ _ _ _ | _..--'-. -\033[37;1m/ ___|| |__ ___ _ __| | ___ ___| |__ >.`__.-\"\"\;\"` -\033[37;1m\___ \| '_ \ / _ \ '__| |/ _ \ / __| |/ / / /( ^\\ -\033[37;1m ___) | | | | __/ | | | (_) | (__| < '-`) =|-. -\033[37;1m|____/|_| |_|\___|_| |_|\___/ \___|_|\_\ /`--.'--' \ .-. -\033[37;1m .'`-._ `.\ | J / -\033[37;1m / `--.| \__/\033[0m""") + print(Fore.WHITE + Style.BRIGHT + +""" .\"\"\"-. + / \\ + ____ _ _ _ | _..--'-. +/ ___|| |__ ___ _ __| | ___ ___| |__ >.`__.-\"\"\;\"` +\___ \| '_ \ / _ \ '__| |/ _ \ / __| |/ / / /( ^\\ + ___) | | | | __/ | | | (_) | (__| < '-`) =|-. +|____/|_| |_|\___|_| |_|\___/ \___|_|\_\ /`--.'--' \ .-. + .'`-._ `.\ | J / + / `--.| \__/""") if args.tor or args.unique_tor: print("Warning: some websites might refuse connecting over TOR, so note that using this option might increase connection errors.") From 459fb29be6ad78edd77de20e293d6f82e79bde6a Mon Sep 17 00:00:00 2001 From: Yahya SayadArbabi Date: Fri, 4 Jan 2019 13:15:17 +0330 Subject: [PATCH 03/14] Fix "Docker notes" --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d66a6921..afc43bd3 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ The ```--rm``` flag is optional. It removes the container filesystem after runn One caveat is the text file that is created will only exist in the container so you will not be able to get at that. -Or you can simply use docker to run `sherlock`: +Or you can simply use "Docker Hub" to run `sherlock`: ``` docker run theyahya/sherlock user123 ``` From f7f36b51376d84f2f7bbb9568caeac06412d12ac Mon Sep 17 00:00:00 2001 From: theodosisathanasakis <32517114+theodosisathanasakis@users.noreply.github.com> Date: Thu, 3 Jan 2019 13:39:33 +0200 Subject: [PATCH 04/14] Install packages for linux The script downloads the packages that are necessary for the sherlock.py. It is an automated procedure instead of the instructions in the README.md. It contains some packages that are not in README but I couldn't run "pip3 install -r requirements.txt" without them. It is for linux, I don't know if it works on something else. I believe it is useful! --- install_packages.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 install_packages.sh diff --git a/install_packages.sh b/install_packages.sh new file mode 100644 index 00000000..cd6eb25d --- /dev/null +++ b/install_packages.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# install python3 if not exist +sudo apt-get install python3 + +# install the all the necessery packages and requirements +sudo apt-get install python3-pip +sudo pip3 install --upgrade setuptools +sudo pip3 install -r requirements.txt + + From de1d0d643cb01a0956dc1678e1476dcd84fc105a Mon Sep 17 00:00:00 2001 From: Yahya SayadArbabi Date: Fri, 4 Jan 2019 19:34:00 +0330 Subject: [PATCH 05/14] bump version --- sherlock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherlock.py b/sherlock.py index 5a0145d0..8d412afe 100644 --- a/sherlock.py +++ b/sherlock.py @@ -21,7 +21,7 @@ from requests_futures.sessions import FuturesSession from torrequest import TorRequest module_name = "Sherlock: Find Usernames Across Social Networks" -__version__ = "0.1.3" +__version__ = "0.1.5" amount=0 # TODO: fix tumblr From f1f356d9d220fe0606128ee2d1d887f77c4c2bc0 Mon Sep 17 00:00:00 2001 From: theodosisathanasakis Date: Fri, 4 Jan 2019 19:56:17 +0200 Subject: [PATCH 06/14] Right regex for Facebook-new --- data.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data.json b/data.json index 0ff11370..cbebd700 100644 --- a/data.json +++ b/data.json @@ -15,7 +15,7 @@ "url": "https://www.facebook.com/{}", "urlMain": "https://www.facebook.com/", "errorType": "status_code", - "regexCheck": "^([a-zA-Z0-9.]){5,}$" + "regexCheck": "^a-zA-Z0-9{4,49}(? Date: Sat, 5 Jan 2019 00:45:54 +0330 Subject: [PATCH 07/14] Add CODE_OF_CONDUCT.md --- CODE_OF_CONDUCT.md | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..f54c40d5 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at yahya.arbabi@gmail.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq From 5f4df01e97be115e8b55e41f5a6e6060414da5a3 Mon Sep 17 00:00:00 2001 From: Yahya SayadArbabi Date: Sat, 5 Jan 2019 01:19:15 +0330 Subject: [PATCH 08/14] Fix the lenght --- data.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data.json b/data.json index cbebd700..018b65b9 100644 --- a/data.json +++ b/data.json @@ -15,7 +15,7 @@ "url": "https://www.facebook.com/{}", "urlMain": "https://www.facebook.com/", "errorType": "status_code", - "regexCheck": "^a-zA-Z0-9{4,49}(? Date: Sat, 5 Jan 2019 01:20:41 +0330 Subject: [PATCH 09/14] bump version --- sherlock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherlock.py b/sherlock.py index 8d412afe..d3345ad6 100644 --- a/sherlock.py +++ b/sherlock.py @@ -21,7 +21,7 @@ from requests_futures.sessions import FuturesSession from torrequest import TorRequest module_name = "Sherlock: Find Usernames Across Social Networks" -__version__ = "0.1.5" +__version__ = "0.1.6" amount=0 # TODO: fix tumblr From 2791e1553aed311f0de1cba3949e11ae61afdd80 Mon Sep 17 00:00:00 2001 From: nil0x42 Date: Fri, 4 Jan 2019 23:59:58 +0000 Subject: [PATCH 10/14] Fix incorrect file save notification When username argument was provided with uppercase chars, sherlock said that file was saved into "UserName.txt" instead of "username.txt" (fname var) --- sherlock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherlock.py b/sherlock.py index d3345ad6..972e8e46 100644 --- a/sherlock.py +++ b/sherlock.py @@ -287,7 +287,7 @@ def sherlock(username, verbose=False, tor=False, unique_tor=False): print((Style.BRIGHT + Fore.GREEN + "[" + Fore.YELLOW + "*" + Fore.GREEN + "] Saved: " + - Fore.WHITE + "{}").format(username+".txt")) + Fore.WHITE + "{}").format(fname)) final_score(amount, fname) return results_total From e0c5bebb5c3ebc3e839193ca4535c722992db95f Mon Sep 17 00:00:00 2001 From: QuantumSheep Date: Sat, 5 Jan 2019 02:52:32 +0100 Subject: [PATCH 11/14] Add Mastodon to the site list --- data.json | 5 +++++ sites.md | 1 + 2 files changed, 6 insertions(+) diff --git a/data.json b/data.json index ba367ac7..250f3940 100644 --- a/data.json +++ b/data.json @@ -567,5 +567,10 @@ "urlMain": "https://www.wikipedia.org/", "errorType": "message", "errorMsg": "If a page was recently created here, it may not be visible yet because of a delay in updating the database" + }, + "Mastodon": { + "url": "https://mstdn.io/@{}", + "urlMain": "https://mstdn.io", + "errorType": "status_code" } } diff --git a/sites.md b/sites.md index 2a6ea425..b5bb2889 100644 --- a/sites.md +++ b/sites.md @@ -99,3 +99,4 @@ 98. [Letterboxd](https://letterboxd.com/) 99. [Coderwall](https://coderwall.com/) 100. [Wikipedia](https://www.wikipedia.org/) +101. [Mastodon](https://mstdn.io/) From 5e87e416d3a18a4392f1919e4a80633025d1822c Mon Sep 17 00:00:00 2001 From: Nathanael Demacon Date: Sat, 5 Jan 2019 02:56:54 +0100 Subject: [PATCH 12/14] Add a / at the end of urlMain link --- data.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data.json b/data.json index 250f3940..1fedcea4 100644 --- a/data.json +++ b/data.json @@ -570,7 +570,7 @@ }, "Mastodon": { "url": "https://mstdn.io/@{}", - "urlMain": "https://mstdn.io", + "urlMain": "https://mstdn.io/", "errorType": "status_code" } } From cc22d88679ec171e71593020502d5af51040281e Mon Sep 17 00:00:00 2001 From: Yahya SayadArbabi Date: Sat, 5 Jan 2019 09:38:28 +0330 Subject: [PATCH 13/14] bump version --- sherlock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherlock.py b/sherlock.py index 972e8e46..161213e0 100644 --- a/sherlock.py +++ b/sherlock.py @@ -21,7 +21,7 @@ from requests_futures.sessions import FuturesSession from torrequest import TorRequest module_name = "Sherlock: Find Usernames Across Social Networks" -__version__ = "0.1.6" +__version__ = "0.1.7" amount=0 # TODO: fix tumblr From f27d2b2aed83bc350e6daa33b89c5baf96883967 Mon Sep 17 00:00:00 2001 From: Yahya SayadArbabi Date: Sat, 5 Jan 2019 11:14:13 +0330 Subject: [PATCH 14/14] bump version --- sherlock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherlock.py b/sherlock.py index 161213e0..8f0035dd 100644 --- a/sherlock.py +++ b/sherlock.py @@ -21,7 +21,7 @@ from requests_futures.sessions import FuturesSession from torrequest import TorRequest module_name = "Sherlock: Find Usernames Across Social Networks" -__version__ = "0.1.7" +__version__ = "0.1.8" amount=0 # TODO: fix tumblr