Merge pull request #135 from TheYahya/hoadlck-response_url-site-fixes

Response URL Site Detection Strategy Overhaul

data.json

@@ -41,7 +41,7 @@
     "errorMsg":"page not found"
   },
   "Pinterest": {
-    "url": "https://www.pinterest.com/{}",
+    "url": "https://www.pinterest.com/{}/",
     "urlMain": "https://www.pinterest.com/",
     "errorType": "response_url",
     "errorUrl": "https://www.pinterest.com/?show_error=true"
@@ -415,7 +415,7 @@
     "errorType": "status_code"
   },
   "WordPress": {
-    "url": "https://{}.wordpress.com",
+    "url": "https://{}.wordpress.com/",
     "urlMain": "https://wordpress.com",
     "errorType": "response_url",
     "errorUrl": "wordpress.com/typo/?subdomain=",

sherlock.py

@@ -208,13 +208,27 @@ def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False, pr
                 if net_info["errorType"] == 'status_code':
                     request_method = session.head
 
+            if net_info["errorType"] == "response_url":
+                #Site forwards request to a different URL if username not
+                #found.  Disallow the redirect so we can capture the
+                #http status from the original URL request.
+                allow_redirects = False
+            else:
+                #Allow whatever redirect that the site wants to do.
+                #The final result of the request will be what is available.
+                allow_redirects = True
+
             # This future starts running the request in a new thread, doesn't block the main thread
             if proxy != None:
                 proxies = {"http": proxy, "https": proxy}
-                future = request_method(
+                future = request_method(url=url, headers=headers,
-                    url=url, headers=headers, proxies=proxies)
+                                        proxies=proxies,
+                                        allow_redirects=allow_redirects
+                                       )
             else:
-                future = request_method(url=url, headers=headers)
+                future = request_method(url=url, headers=headers,
+                                        allow_redirects=allow_redirects
+                                       )
 
             # Store future in data for access later
             net_info["request_future"] = future
@@ -290,9 +304,13 @@ def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False, pr
                 exists = "no"
 
         elif error_type == "response_url":
-            error = net_info.get("errorUrl")
+            # For this detection method, we have turned off the redirect.
-            # Checks if the redirect url is the same as the one defined in data.json
+            # So, there is no need to check the response URL: it will always
-            if not error in r.url:
+            # match the request.  Instead, we will ensure that the response
+            # code indicates that the request was successful (i.e. no 404, or
+            # forward to some odd redirect).
+            if (r.status_code >= 200) and (r.status_code < 300):
+                #
                 print_found(social_network, url, response_time, verbose)
                 write_to_file(url, f)
                 exists = "yes"

tests/all.py

@@ -23,7 +23,7 @@ class SherlockDetectTests(SherlockBaseTest):
         """
 
         self.username_check(['jack'],  ['Twitter'],   exist_check=True)
-        #self.username_check(['dfox'],  ['devRant'],   exist_check=True)
+        self.username_check(['dfox'],  ['devRant'],   exist_check=True)
         self.username_check(['blue'],  ['Pinterest'], exist_check=True)
         self.username_check(['kevin'], ['Instagram'], exist_check=True)
         self.username_check(['zuck'],  ['Facebook'],  exist_check=True)
@@ -92,3 +92,51 @@ class SherlockDetectTests(SherlockBaseTest):
                            )
 
         return
+
+
+class SherlockSiteCoverageTests(SherlockBaseTest):
+    def test_coverage_false_via_response_url(self):
+        """Test Username Does Not Exist Site Coverage (Via Response URL).
+
+        This test checks all sites with the "response URL" detection mechanism
+        to ensure that a Username that does not exist is reported that way.
+
+        Keyword Arguments:
+        self                   -- This object.
+
+        Return Value:
+        N/A.
+        Will trigger an assert if detection mechanism did not work as expected.
+        """
+
+        self.username_check(['noonewouldeverusethis7'],
+                            ["Pinterest", "iMGSRC.RU", "Pastebin",
+                             "WordPress", "devRant", "ImageShack", "MeetMe"
+                            ],
+                            exist_check=False
+                           )
+
+        return
+
+    def test_coverage_true_via_response_url(self):
+        """Test Username Does Exist Site Coverage (Via Response URL).
+
+        This test checks all sites with the "response URL" detection mechanism
+        to ensure that a Username that does exist is reported that way.
+
+        Keyword Arguments:
+        self                   -- This object.
+
+        Return Value:
+        N/A.
+        Will trigger an assert if detection mechanism did not work as expected.
+        """
+
+        self.username_check(['blue'],
+                            ["Pinterest", "iMGSRC.RU", "Pastebin",
+                             "WordPress", "devRant", "ImageShack", "MeetMe"
+                            ],
+                            exist_check=True
+                           )
+
+        return