diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..0de6f864 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,25 @@ +group: travis_latest +language: python +cache: pip +matrix: + allow_failures: + - python: nightly + include: + - python: 3.6 + - python: 3.7 + dist: xenial # required for Python >= 3.7 (travis-ci/travis-ci#9069) + - python: nightly + dist: xenial +install: + - pip install -r requirements.txt + - pip install flake8 +before_script: + # stop the build if there are Python syntax errors or undefined names + - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics +script: + - python3 -m unittest tests.all --buffer --verbose +notifications: + on_success: change + on_failure: change # `always` will be the setting once code changes slow down diff --git a/README.md b/README.md index 2e945e8f..7189ad1c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -# Sherlock -> Find usernames across [social networks](https://github.com/sdushantha/sherlock/blob/master/sites.md) +# Sherlock [![Build Status](https://travis-ci.com/TheYahya/sherlock.svg?branch=master)](https://travis-ci.com/TheYahya/sherlock) +> Find usernames across [social networks](https://github.com/theyahya/sherlock/blob/master/sites.md) [![asciicast](https://asciinema.org/a/IMRMmbAxFGUgd2SJN0rkAfaPb.svg)](https://asciinema.org/a/IMRMmbAxFGUgd2SJN0rkAfaPb) @@ -14,6 +14,8 @@ $ git clone https://github.com/TheYahya/sherlock.git # change the working directory to sherlock $ cd sherlock +# install python3 and python3-pip if not exist + # install the requirements $ pip3 install -r requirements.txt ``` @@ -63,7 +65,7 @@ Once the image is built sherlock can be invoked by running the following: docker run --rm mysherlock-image user123 ``` -The ```--rm``` flag is optional. It removes the container filesystem after running so you do not have a bunch of leftover container filesystem cruft. See https://docs.docker.com/engine/reference/run/#clean-up---rm +The optional ```--rm``` flag removes the container filesystem on completion to prevent cruft build-up. See https://docs.docker.com/engine/reference/run/#clean-up---rm One caveat is the text file that is created will only exist in the container so you will not be able to get at that. @@ -73,7 +75,26 @@ Or you can simply use "Docker Hub" to run `sherlock`: docker run theyahya/sherlock user123 ``` +## Tests +If you are contributing to Sherlock, then Thank You! + +Before creating a pull request with new development, please run the tests +to ensure that all is well. It would also be a good idea to run the tests +before starting development to distinguish problems between your +environment and the Sherlock software. + +The following is an example of the command line to run all the tests for +Sherlock. This invocation hides the progress text that Sherlock normally +outputs, and instead shows the verbose output of the tests. + +``` +% python3 -m unittest tests.all --buffer --verbose +``` + +Note that the tests are very much a work in progress. Significant work is +required to get full test coverage. But, the current tests are working +properly, and will be expanded as time goes by. + ## License -MIT License -Copyright (c) 2018 Siddharth Dushantha +MIT © [Yahya SayadArbabi](https://theyahya.com) diff --git a/data.json b/data.json index a63b8503..f47095fc 100644 --- a/data.json +++ b/data.json @@ -666,5 +666,87 @@ "url": "https://{}.itch.io/", "urlMain": "https://itch.io/", "errorType": "status_code" + }, + "Basecamp":{ + "url": "https://{}.basecamphq.com", + "urlMain": "https://basecamp.com/", + "errorType": "message", + "errorMsg": "The account you were looking for doesn't exist" + }, + "ProductHunt":{ + "url": "https://www.producthunt.com/@{}", + "urlMain": "https://www.producthunt.com/", + "errorType": "message", + "errorMsg": "Product Hunt is a curation of the best new products" + }, + "Younow":{ + "url": "https://www.younow.com/{}", + "urlMain": "https://www.younow.com/", + "errorType": "message", + "errorMsg": "pageTitle || 'YouNow - Broadcast Live" + }, + "Smashcast":{ + "url": "https://www.smashcast.tv/api/media/live/{}", + "urlMain": "https://www.smashcast.tv/", + "errorType": "status_code" + }, + "AskFM":{ + "url": "https://ask.fm/{}", + "urlMain": "https://ask.fm/", + "errorType": "status_code" + }, + "KanoWorld":{ + "url": "https://api.kano.me/progress/user/{}", + "urlMain": "https://world.kano.me/", + "errorType": "status_code" + }, + "EyeEm":{ + "url": "https://www.eyeem.com/u/{}", + "urlMain": "https://www.eyeem.com/", + "errorType": "status_code" + }, + "Wikia":{ + "url": "https://wikia.com/wiki/User:{}", + "urlMain": "http://www.wikia.com/", + "errorType": "message", + "errorMsg": "does not exist" + }, + "CreativeMarket":{ + "url": "https://creativemarket.com/{}", + "urlMain": "https://creativemarket.com/", + "errorType": "status_code" + }, + "Venmo":{ + "url": "https://venmo.com/{}", + "urlMain": "https://venmo.com/", + "errorType": "status_code" + }, + "HubPages":{ + "url": "https://hubpages.com/@{}", + "urlMain": "https://hubpages.com/", + "errorType": "status_code" + }, + "StreamMe":{ + "url": "https://www.stream.me/{}", + "urlMain": "https://www.stream.me/", + "errorType": "status_code" + }, + "Kik":{ + "url": "https://ws2.kik.com/user/{}", + "urlMain": "http://kik.me/", + "errorType": "message", + "errorMsg": "The page you requested was not found" + }, + "Star Citizen": { + "url": "https://robertsspaceindustries.com/citizens/{}", + "urlMain": "https://robertsspaceindustries.com/", + "errorType": "status_code" + }, + "EVE Online": { + "url": "https://evewho.com/search/{}", + "urlMain": "https://eveonline.com", + "errorType": "message", + "errorMsg": "No results found with your search..." } } + diff --git a/install_packages.sh b/install_packages.sh old mode 100644 new mode 100755 index cd6eb25d..e444571f --- a/install_packages.sh +++ b/install_packages.sh @@ -1,11 +1,51 @@ #!/bin/bash -# install python3 if not exist -sudo apt-get install python3 +# Determine which is the default package manager +APT=$(which apt) +PACMAN=$(which pacman) +DNF=$(which dnf) +YUM=$(which yum) +ZYPPER=$(which zypper) +# install python3 and pip3 if not exist +if [ ${#APT} -gt 0 ]; then + sudo apt-get install python3 + sudo apt-get install python3-pip +elif [ ${#PACMAN} -gt 0 ]; then + sudo pacman -S python3 + sudo pacman -S python3-pip +elif [ ${#DNF} -gt 0 ]; then + sudo dnf install python3 + sudo dnf install python3-pip +elif [ ${#YUM} -gt 0 ]; then + sudo yum install python3 + sudo yum install python3-pip +elif [ ${#ZYPPER} -gt 0 ]; then + sudo zypper install python3 + sudo zypper install python3-pip +else + echo "Unknown package manager. Download one of the following:" + echo " apt, pacman, dnf, yum or zypper" + echo "" + echo "or use README.md for instructions." + exit 1 +fi + + # install the all the necessery packages and requirements -sudo apt-get install python3-pip -sudo pip3 install --upgrade setuptools -sudo pip3 install -r requirements.txt +echo '' +echo '' +while true; do + echo 'Do you want dependencies to be installed globally (or locally) [Y/n]?' + read ans + if [[ ${#ans} -eq 0 || $ans = "Y" || $ans = "y" ]]; then + sudo pip3 install --upgrade setuptools + sudo pip3 install -r requirements.txt + elif [[ $ans = "N" || $ans = "n" ]]; then + sudo pip3 install --user --upgrade setuptools + sudo pip3 install --user -r requirements.txt + fi + [[ ${#ans} -eq 0 || $ans = "Y" || $ans = "y" || $ans = "N" || $ans = "n" ]] && break; +done diff --git a/sherlock.py b/sherlock.py index 819e057d..e7fef9d7 100644 --- a/sherlock.py +++ b/sherlock.py @@ -23,7 +23,7 @@ from requests_futures.sessions import FuturesSession from torrequest import TorRequest module_name = "Sherlock: Find Usernames Across Social Networks" -__version__ = "0.2.6" +__version__ = "0.3.0" amount = 0 # TODO: fix tumblr @@ -326,6 +326,7 @@ def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False, pr Fore.WHITE + "{}").format(fname)) final_score(amount, f) + f.close() return results_total diff --git a/sites.md b/sites.md index 3f60d044..6b834284 100644 --- a/sites.md +++ b/sites.md @@ -1,4 +1,4 @@ -## List Of Supported Sites (118 Sites In Total!) +## List Of Supported Sites (133 Sites In Total!) 1. [Instagram](https://www.instagram.com/) 2. [Twitter](https://www.twitter.com/) 3. [Facebook](https://www.facebook.com/) @@ -114,6 +114,21 @@ 113. [Wikipedia](https://www.wikipedia.org/) 114. [Mastodon](https://mstdn.io/) 115. [Telegram](https://t.me/) -116. [TradingView] (https://www.tradingview.com/) -117. [Kaggle] (https://www.kaggle.com/) -118. [Itch.io] (https://itch.io/) \ No newline at end of file +116. [TradingView](https://www.tradingview.com/) +117. [Kaggle](https://www.kaggle.com/) +118. [Itch.io](https://itch.io/) +119. [Basecamp](https://basecamp.com/) +120. [ProductHunt](https://www.producthunt.com/) +121. [Younow](https://www.younow.com/) +122. [Smashcast](https://www.smashcast.tv/) +123. [AskFM](https://ask.fm/) +124. [KanoWorld](https://world.kano.me/) +125. [EyeEm](https://www.eyeem.com/) +126. [Wikia](http://www.wikia.com/) +127. [CreativeMarket](https://creativemarket.com/) +128. [Venmo](https://venmo.com/) +129. [HubPages](https://hubpages.com/) +130. [StreamMe](https://www.stream.me/) +131. [Kik](http://kik.me/) +132. [Star Citizen](https://robertsspaceindustries.com/) +133. [EVE Online](https://eveonline.com) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..944e27ce --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,4 @@ +"""Sherlock Tests + +This package contains various submodules used to run tests. +""" diff --git a/tests/all.py b/tests/all.py new file mode 100644 index 00000000..87d3b9b8 --- /dev/null +++ b/tests/all.py @@ -0,0 +1,94 @@ +"""Sherlock Tests + +This module contains various tests. +""" +from tests.base import SherlockBaseTest +import unittest + + +class SherlockDetectTests(SherlockBaseTest): + def test_detect_true(self): + """Test Username Existence Detection. + + This test ensures that the mechanism of ensuring that a Username + exists works properly. + + Keyword Arguments: + self -- This object. + + Return Value: + N/A. + Will trigger an assert if Usernames which are known to exist are + not detected. + """ + + self.username_check(['jack'], ['Twitter'], exist_check=True) + #self.username_check(['dfox'], ['devRant'], exist_check=True) + self.username_check(['blue'], ['Pinterest'], exist_check=True) + self.username_check(['kevin'], ['Instagram'], exist_check=True) + self.username_check(['zuck'], ['Facebook'], exist_check=True) + + return + + def test_detect_false_via_message(self): + """Test Username Does Not Exist (Via Message). + + This test ensures that the "message" detection mechanism of + ensuring that a Username does *not* exist works properly. + + Keyword Arguments: + self -- This object. + + Return Value: + N/A. + Will trigger an assert if detection mechanism did not work as expected. + """ + + self.username_check(['jackkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk'], + ['Instagram'], + exist_check=False + ) + + return + + def test_detect_false_via_status_code(self): + """Test Username Does Not Exist (Via Status Code). + + This test ensures that the "status code" detection mechanism of + ensuring that a Username does *not* exist works properly. + + Keyword Arguments: + self -- This object. + + Return Value: + N/A. + Will trigger an assert if detection mechanism did not work as expected. + """ + + self.username_check(['jackkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk'], + ['Facebook'], + exist_check=False + ) + + return + + def test_detect_false_via_response_url(self): + """Test Username Does Not Exist (Via Response URL). + + This test ensures that the "response URL" detection mechanism of + ensuring that a Username does *not* exist works properly. + + Keyword Arguments: + self -- This object. + + Return Value: + N/A. + Will trigger an assert if detection mechanism did not work as expected. + """ + + self.username_check(['jackkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk'], + ['Pinterest'], + exist_check=False + ) + + return diff --git a/tests/base.py b/tests/base.py new file mode 100644 index 00000000..0f992ba8 --- /dev/null +++ b/tests/base.py @@ -0,0 +1,107 @@ +"""Sherlock Base Tests + +This module contains various utilities for running tests. +""" +import json +import os +import os.path +import unittest +import sherlock +import warnings + + +class SherlockBaseTest(unittest.TestCase): + def setUp(self): + """Sherlock Base Test Setup. + + Does common setup tasks for base Sherlock tests. + + Keyword Arguments: + self -- This object. + + Return Value: + N/A. + """ + + #This ignores the ResourceWarning from an unclosed SSLSocket. + #TODO: Figure out how to fix the code so this is not needed. + warnings.simplefilter("ignore", ResourceWarning) + + # Load the data file with all site information. + data_file_path = os.path.join(os.path.dirname(os.path.realpath(sherlock.__file__)), "data.json") + with open(data_file_path, "r", encoding="utf-8") as raw: + self.site_data_all = json.load(raw) + + self.verbose=False + self.tor=False + self.unique_tor=False + + return + + def site_data_filter(self, site_list): + """Filter Site Data. + + Keyword Arguments: + self -- This object. + site_list -- List of strings corresponding to sites which + should be filtered. + + Return Value: + Dictionary containing sub-set of site data specified by 'site_list'. + """ + + # Create new dictionary that has filtered site data based on input. + # Note that any site specified which is not understood will generate + # an error. + site_data = {} + for site in site_list: + with self.subTest(f"Checking test vector Site '{site}' " + f"exists in total site data." + ): + site_data[site] = self.site_data_all[site] + + return site_data + + def username_check(self, username_list, site_list, exist_check=True): + """Username Exist Check. + + Keyword Arguments: + self -- This object. + username_list -- List of strings corresponding to usernames + which should exist on *all* of the sites. + site_list -- List of strings corresponding to sites which + should be filtered. + exist_check -- Boolean which indicates if this should be + a check for Username existence, + or non-existence. + + Return Value: + N/A. + Will trigger an assert if Username does not have the expected + existence state. + """ + + #Filter all site data down to just what is needed for this test. + site_data = self.site_data_filter(site_list) + + if exist_check: + check_type_text = "exists" + exist_result_desired = "yes" + else: + check_type_text = "does not exist" + exist_result_desired = "no" + + for username in username_list: + results = sherlock.sherlock(username, + site_data, + verbose=self.verbose, + tor=self.tor, + unique_tor=self.unique_tor + ) + for site, result in results.items(): + with self.subTest(f"Checking Username '{username}' " + f"{check_type_text} on Site '{site}'" + ): + self.assertEqual(result['exists'], exist_result_desired) + + return