From be59b9110753c788a430a0e1632fbd563bd07ea3 Mon Sep 17 00:00:00 2001 From: "Christopher K. Hoadley" Date: Sat, 2 Feb 2019 20:32:06 -0600 Subject: [PATCH 1/4] Add information about a claimed and an unclaimed username to the site information. This will allow the tests to directly source the information from the JSON data. It will also allow people who add new sites to also add test data (which will automatically get tested). Add a new test method which finds all sites of a given detect algorithm, and which also has test vectors, and runs tests against them. At the current time, the test vectors are optional. But, once when there is finally complete coverage, then they will be required. Anyone who adds a site without also adding test data will trigger a failure. --- data.json | 4 +++- tests/all.py | 12 ++---------- tests/base.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/data.json b/data.json index ed7b9fb9..2b33fab6 100644 --- a/data.json +++ b/data.json @@ -221,7 +221,9 @@ "rank": 921, "regexCheck": "^[a-zA-Z][a-zA-Z0-9_-]*$", "url": "https://dribbble.com/{}", - "urlMain": "https://dribbble.com/" + "urlMain": "https://dribbble.com/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "EVE Online": { "errorType": "response_url", diff --git a/tests/all.py b/tests/all.py index e21e960f..0f26dbde 100644 --- a/tests/all.py +++ b/tests/all.py @@ -205,11 +205,7 @@ class SherlockSiteCoverageTests(SherlockBaseTest): Will trigger an assert if detection mechanism did not work as expected. """ - self.username_check(['noonewouldeverusethis7'], - ["Dribbble" - ], - exist_check=False - ) + self.detect_type_check("message", exist_check=False) return @@ -227,10 +223,6 @@ class SherlockSiteCoverageTests(SherlockBaseTest): Will trigger an assert if detection mechanism did not work as expected. """ - self.username_check(['blue'], - ["Dribbble" - ], - exist_check=True - ) + self.detect_type_check("message", exist_check=True) return diff --git a/tests/base.py b/tests/base.py index 0f992ba8..e17d8c8d 100644 --- a/tests/base.py +++ b/tests/base.py @@ -105,3 +105,50 @@ class SherlockBaseTest(unittest.TestCase): self.assertEqual(result['exists'], exist_result_desired) return + + def detect_type_check(self, detect_type, exist_check=True): + """Username Exist Check. + + Keyword Arguments: + self -- This object. + detect_type -- String corresponding to detection algorithm + which is desired to be tested. + Note that only sites which have documented + usernames which exist and do not exist + will be tested. + exist_check -- Boolean which indicates if this should be + a check for Username existence, + or non-existence. + + Return Value: + N/A. + Runs tests on all sites using the indicated detection algorithm + and which also has test vectors specified. + Will trigger an assert if Username does not have the expected + existence state. + """ + + for site, site_data in self.site_data_all.items(): + if ( + (site_data["errorType"] != detect_type) or + (site_data.get("username_claimed") is None) or + (site_data.get("username_unclaimed") is None) + ): + # This is either not a site we are interested in, or the + # site does not contain the required information to do + # the tests. + pass + else: + # We should run a test on this site. + + # Figure out which type of user + if exist_check: + username_list = [site_data.get("username_claimed")] + else: + username_list = [site_data.get("username_unclaimed")] + self.username_check(username_list, + [site], + exist_check=exist_check + ) + + return From a576af358fbe36139c122b1730906e69a8d9044b Mon Sep 17 00:00:00 2001 From: "Christopher K. Hoadley" Date: Sat, 2 Feb 2019 20:40:16 -0600 Subject: [PATCH 2/4] Convert all supported HTTP Status Code sites to use JSON driven tests. --- data.json | 44 +++++++++++++++++++++++++++++++++----------- tests/all.py | 16 ++-------------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/data.json b/data.json index 2b33fab6..f263bc2f 100644 --- a/data.json +++ b/data.json @@ -10,25 +10,33 @@ "errorType": "status_code", "rank": 331, "url": "https://9gag.com/u/{}", - "urlMain": "https://9gag.com/" + "urlMain": "https://9gag.com/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "About.me": { "errorType": "status_code", "rank": 12674, "url": "https://about.me/{}", - "urlMain": "https://about.me/" + "urlMain": "https://about.me/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "Academia.edu": { "errorType": "status_code", "rank": 383, "url": "https://independent.academia.edu/{}", - "urlMain": "https://www.academia.edu/" + "urlMain": "https://www.academia.edu/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "AngelList": { "errorType": "status_code", "rank": 3371, "url": "https://angel.co/{}", - "urlMain": "https://angel.co/" + "urlMain": "https://angel.co/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "Aptoide": { "errorType": "status_code", @@ -46,7 +54,9 @@ "errorType": "status_code", "rank": 282942, "url": "https://blip.fm/{}", - "urlMain": "https://blip.fm/" + "urlMain": "https://blip.fm/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "Badoo": { "errorType": "status_code", @@ -58,7 +68,9 @@ "errorType": "status_code", "rank": 568, "url": "https://www.bandcamp.com/{}", - "urlMain": "https://www.bandcamp.com/" + "urlMain": "https://www.bandcamp.com/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "Basecamp": { "errorMsg": "The account you were looking for doesn't exist", @@ -71,7 +83,9 @@ "errorType": "status_code", "rank": 401, "url": "https://www.behance.net/{}", - "urlMain": "https://www.behance.net/" + "urlMain": "https://www.behance.net/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "BitBucket": { "errorType": "status_code", @@ -97,7 +111,9 @@ "errorType": "status_code", "rank": 294, "url": "https://buzzfeed.com/{}", - "urlMain": "https://buzzfeed.com/" + "urlMain": "https://buzzfeed.com/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "Canva": { "errorType": "response_url", @@ -129,13 +145,17 @@ "errorType": "status_code", "rank": 2325, "url": "https://www.codecademy.com/{}", - "urlMain": "https://www.codecademy.com/" + "urlMain": "https://www.codecademy.com/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "Codementor": { "errorType": "status_code", "rank": 12164, "url": "https://www.codementor.io/{}", - "urlMain": "https://www.codementor.io/" + "urlMain": "https://www.codementor.io/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "Codepen": { "errorType": "status_code", @@ -200,7 +220,9 @@ "errorType": "status_code", "rank": 24423, "url": "https://www.designspiration.net/{}/", - "urlMain": "https://www.designspiration.net/" + "urlMain": "https://www.designspiration.net/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "DeviantART": { "errorType": "status_code", diff --git a/tests/all.py b/tests/all.py index 0f26dbde..f394fe7c 100644 --- a/tests/all.py +++ b/tests/all.py @@ -157,13 +157,7 @@ class SherlockSiteCoverageTests(SherlockBaseTest): Will trigger an assert if detection mechanism did not work as expected. """ - self.username_check(['noonewouldeverusethis7'], - ["Academia.edu", "9GAG", "About.me", "AngelList", - "BLIP.fm", "Bandcamp", "Behance", "BuzzFeed", - "Codecademy", "Codementor", "Designspiration" - ], - exist_check=False - ) + self.detect_type_check("status_code", exist_check=False) return @@ -181,13 +175,7 @@ class SherlockSiteCoverageTests(SherlockBaseTest): Will trigger an assert if detection mechanism did not work as expected. """ - self.username_check(['blue'], - ["Academia.edu", "9GAG", "About.me", "AngelList", - "BLIP.fm", "Bandcamp", "Behance", "BuzzFeed", - "Codecademy", "Codementor", "Designspiration" - ], - exist_check=True - ) + self.detect_type_check("status_code", exist_check=True) return From bf0959d84bea25cdfcb6c83be484ad3c55ef1a78 Mon Sep 17 00:00:00 2001 From: "Christopher K. Hoadley" Date: Sat, 2 Feb 2019 20:46:53 -0600 Subject: [PATCH 3/4] Convert all supported Response URL sites to use JSON driven tests. --- data.json | 44 +++++++++++++++++++++++++++++++++----------- tests/all.py | 16 ++-------------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/data.json b/data.json index f263bc2f..7081f74b 100644 --- a/data.json +++ b/data.json @@ -120,7 +120,9 @@ "errorUrl": "https://www.canva.com/{}", "rank": 214, "url": "https://www.canva.com/{}", - "urlMain": "https://www.canva.com/" + "urlMain": "https://www.canva.com/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "Carbonmade": { "errorMsg": "You've accidentally stumbled upon Mike's super secret nap grotto.", @@ -196,7 +198,9 @@ "errorUrl": "https://www.creativemarket.com/", "rank": 1790, "url": "https://creativemarket.com/{}", - "urlMain": "https://creativemarket.com/" + "urlMain": "https://creativemarket.com/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "Crevado": { "errorType": "status_code", @@ -252,7 +256,9 @@ "errorUrl": "https://eveonline.com", "rank": 11650, "url": "https://evewho.com/pilot/{}/", - "urlMain": "https://eveonline.com" + "urlMain": "https://eveonline.com", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "Ebay": { "errorMsg": "The User ID you entered was not found", @@ -279,7 +285,9 @@ "errorUrl": "https://www.eyeem.com/", "rank": 33324, "url": "https://www.eyeem.com/u/{}", - "urlMain": "https://www.eyeem.com/" + "urlMain": "https://www.eyeem.com/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "Facebook": { "errorType": "status_code", @@ -411,7 +419,9 @@ "errorUrl": "https://imageshack.us/", "rank": 37337, "url": "https://imageshack.us/user/{}", - "urlMain": "https://imageshack.us/" + "urlMain": "https://imageshack.us/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "Imgur": { "errorType": "status_code", @@ -529,7 +539,9 @@ "errorUrl": "https://www.meetme.com/", "rank": 14599, "url": "https://www.meetme.com/{}", - "urlMain": "https://www.meetme.com/" + "urlMain": "https://www.meetme.com/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "MixCloud": { "errorMsg": "Page Not Found", @@ -563,7 +575,9 @@ "errorUrl": "https://pastebin.com/index", "rank": 1044, "url": "https://pastebin.com/u/{}", - "urlMain": "https://pastebin.com/" + "urlMain": "https://pastebin.com/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "Patreon": { "errorType": "status_code", @@ -589,7 +603,9 @@ "errorUrl": "https://www.pinterest.com/?show_error=true", "rank": 73, "url": "https://www.pinterest.com/{}/", - "urlMain": "https://www.pinterest.com/" + "urlMain": "https://www.pinterest.com/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "Pixabay": { "errorType": "status_code", @@ -860,7 +876,9 @@ "rank": 60, "regexCheck": "^[a-zA-Z][a-zA-Z0-9_-]*$", "url": "https://{}.wordpress.com/", - "urlMain": "https://wordpress.com" + "urlMain": "https://wordpress.com", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "YouPic": { "errorType": "status_code", @@ -894,14 +912,18 @@ "errorUrl": "https://devrant.com/", "rank": 146159, "url": "https://devrant.com/users/{}", - "urlMain": "https://devrant.com/" + "urlMain": "https://devrant.com/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "iMGSRC.RU": { "errorType": "response_url", "errorUrl": "https://imgsrc.ru/", "rank": 3660, "url": "https://imgsrc.ru/main/user.php?user={}", - "urlMain": "https://imgsrc.ru/" + "urlMain": "https://imgsrc.ru/", + "username_claimed": "blue", + "username_unclaimed": "noonewouldeverusethis7" }, "last.fm": { "errorMsg": "Whoops! Sorry, but this page doesn't exist.", diff --git a/tests/all.py b/tests/all.py index f394fe7c..1fdfec6b 100644 --- a/tests/all.py +++ b/tests/all.py @@ -109,13 +109,7 @@ class SherlockSiteCoverageTests(SherlockBaseTest): Will trigger an assert if detection mechanism did not work as expected. """ - self.username_check(['noonewouldeverusethis7'], - ["Pinterest", "iMGSRC.RU", "Pastebin", - "WordPress", "devRant", "ImageShack", "MeetMe", - "EyeEm", "CreativeMarket", "EVE Online", "Canva" - ], - exist_check=False - ) + self.detect_type_check("response_url", exist_check=False) return @@ -133,13 +127,7 @@ class SherlockSiteCoverageTests(SherlockBaseTest): Will trigger an assert if detection mechanism did not work as expected. """ - self.username_check(['blue'], - ["Pinterest", "iMGSRC.RU", "Pastebin", - "WordPress", "devRant", "ImageShack", "MeetMe", - "EyeEm", "CreativeMarket", "EVE Online", "Canva" - ], - exist_check=True - ) + self.detect_type_check("response_url", exist_check=True) return From ea90e7fa956fa5c44e3ebf04c9e58582094d1d2a Mon Sep 17 00:00:00 2001 From: "Christopher K. Hoadley" Date: Sat, 2 Feb 2019 21:24:46 -0600 Subject: [PATCH 4/4] Bump version and add link to Wiki on procedure to add new sites. --- README.md | 8 +++++++- sherlock.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 474a68bf..a455f14e 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ usage: sherlock.py [-h] [--version] [--verbose] [--rank] [--proxy PROXY_URL] [--json JSON_FILE] USERNAMES [USERNAMES ...] -Sherlock: Find Usernames Across Social Networks (Version 0.4.0) +Sherlock: Find Usernames Across Social Networks (Version 0.5.0) positional arguments: USERNAMES One or more usernames to check with social networks. @@ -94,6 +94,12 @@ Or you can simply use "Docker Hub" to run `sherlock`: docker run theyahya/sherlock user123 ``` +## Adding New Sites + +Please look at the Wiki entry on +[adding new sites](https://github.com/TheYahya/sherlock/wiki/Adding-Sites-To-Sherlock) +to understand the issues. + ## Tests If you are contributing to Sherlock, then Thank You! diff --git a/sherlock.py b/sherlock.py index e7a44728..9d63f9c9 100644 --- a/sherlock.py +++ b/sherlock.py @@ -26,7 +26,7 @@ from torrequest import TorRequest from load_proxies import load_proxies_from_csv, check_proxy_list module_name = "Sherlock: Find Usernames Across Social Networks" -__version__ = "0.4.6" +__version__ = "0.5.0" amount = 0 BANNER = r'''