Merge pull request #1205 from SethFalco/post-requests

feat: support post requests

removed_sites.json

@@ -645,14 +645,6 @@
     "username_claimed": "TheMorozko",
     "username_unclaimed": "noonewouldeverusethis7"
   },
-  "Anilist": {
-    "errorType": "status_code",
-    "regexCheck": "^[A-Za-z0-9]{2,20}$",
-    "url": "https://anilist.co/user/{}/",
-    "urlMain": "https://anilist.co/",
-    "username_claimed": "Josh",
-    "username_unclaimed": "noonewouldeverusethi"
-  },
   "Coil": {
     "errorMsg": "Whoops, the thing you were looking for isn't here",
     "errorType": "message",

removed_sites.md

@@ -1254,20 +1254,6 @@ As of 2021-11-21, 1337x seems to be down causing false positives.
   }
 ```
 
-### Anilist
-As of 2021-11-26, Anilist is returning false positives.
-
-```
-  "Anilist": {
-    "errorType": "status_code",
-    "regexCheck": "^[A-Za-z0-9]{2,20}$",
-    "url": "https://anilist.co/user/{}/",
-    "urlMain": "https://anilist.co/",
-    "username_claimed": "Josh",
-    "username_unclaimed": "noonewouldeverusethi"
-  }
-```
-
 ### Coil
 As of 2021-11-26, Coil is returning false positives.
 

sherlock/resources/data.json

@@ -65,6 +65,22 @@
     "username_claimed": "blue",
     "username_unclaimed": "noonewouldeverusethis7"
   },
+  "Anilist": {
+    "errorType": "status_code",
+    "regexCheck": "^[A-Za-z0-9]{2,20}$",
+    "request_method": "POST",
+    "request_payload": {
+      "query": "query($name:String){User(name:$name){id}}",
+      "variables": {
+        "name": "{}"
+      }
+    },
+    "url": "https://anilist.co/user/{}/",
+    "urlProbe": "https://graphql.anilist.co/",
+    "urlMain": "https://anilist.co/",
+    "username_claimed": "Josh",
+    "username_unclaimed": "noonewouldeverusethi"
+  },
   "Apple Developer": {
     "errorType": "status_code",
     "url": "https://developer.apple.com/forums/profile/{}",
@@ -1792,7 +1808,7 @@
   },
   "Vero": {
     "errorType": "status_code",
-    "request_head_only": false,
+    "request_method": "GET",
     "url": "https://vero.co/{}",
     "urlMain": "https://vero.co/",
     "username_claimed": "blue",
@@ -1815,7 +1831,7 @@
   },
   "VirusTotal": {
     "errorType": "status_code",
-    "request_head_only": false,
+    "request_method": "GET",
     "url": "https://www.virustotal.com/ui/users/{}/trusted_users",
     "urlMain": "https://www.virustotal.com/",
     "urlProbe": "https://www.virustotal.com/ui/users/{}/avatar",

sherlock/sherlock.py

@@ -125,6 +125,21 @@ def get_response(request_future, error_type, social_network):
     return response, error_context, expection_text
 
 
+def interpolate_string(object, username):
+    """Insert a string into the string properties of an object recursively."""
+
+    if isinstance(object, str):
+        return object.replace("{}", username)
+    elif isinstance(object, dict):
+        for key, value in object.items():
+            object[key] = interpolate_string(value, username)
+    elif isinstance(object, list):
+        for i in object:
+            object[i] = interpolate_string(object[i], username)
+
+    return object
+
+
 def sherlock(username, site_data, query_notify,
              tor=False, unique_tor=False,
              proxy=None, timeout=None):
@@ -207,7 +222,7 @@ def sherlock(username, site_data, query_notify,
             headers.update(net_info["headers"])
 
         # URL of user on site (if it exists)
-        url = net_info["url"].format(username)
+        url = interpolate_string(net_info["url"], username)
 
         # Don't make request if username is invalid for the site
         regex_check = net_info.get("regexCheck")
@@ -225,25 +240,44 @@ def sherlock(username, site_data, query_notify,
             # URL of user on site (if it exists)
             results_site["url_user"] = url
             url_probe = net_info.get("urlProbe")
+            request_method = net_info.get("request_method")
+            request_payload = net_info.get("request_payload")
+            request = None
+
+            if request_method is not None:
+                if request_method == "GET":
+                    request = session.get
+                elif request_method == "HEAD":
+                    request = session.head
+                elif request_method == "POST":
+                    request = session.post
+                elif request_method == "PUT":
+                    request = session.put
+                else:
+                    raise RuntimeError( f"Unsupported request_method for {url}")
+
+            if request_payload is not None:
+                request_payload = interpolate_string(request_payload, username)
+
             if url_probe is None:
                 # Probe URL is normal one seen by people out on the web.
                 url_probe = url
             else:
                 # There is a special URL for probing existence separate
                 # from where the user profile normally can be found.
-                url_probe = url_probe.format(username)
+                url_probe = interpolate_string(url_probe, username)
 
-            if (net_info["errorType"] == 'status_code' and
+            if request is None:
-                net_info.get("request_head_only", True) == True):
+                if net_info["errorType"] == 'status_code':
                     # In most cases when we are detecting by status code,
                     # it is not necessary to get the entire body:  we can
                     # detect fine with just the HEAD response.
-                request_method = session.head
+                    request = session.head
                 else:
                     # Either this detect method needs the content associated
                     # with the GET response, or this specific website will
                     # not respond properly unless we request the whole page.
-                request_method = session.get
+                    request = session.get
 
             if net_info["errorType"] == "response_url":
                 # Site forwards request to a different URL if username not
@@ -258,15 +292,17 @@ def sherlock(username, site_data, query_notify,
             # This future starts running the request in a new thread, doesn't block the main thread
             if proxy is not None:
                 proxies = {"http": proxy, "https": proxy}
-                future = request_method(url=url_probe, headers=headers,
+                future = request(url=url_probe, headers=headers,
                                  proxies=proxies,
                                  allow_redirects=allow_redirects,
-                                        timeout=timeout
+                                 timeout=timeout,
+                                 json=request_payload
                                  )
             else:
-                future = request_method(url=url_probe, headers=headers,
+                future = request(url=url_probe, headers=headers,
                                 allow_redirects=allow_redirects,
-                                        timeout=timeout
+                                timeout=timeout,
+                                json=request_payload
                                 )
 
             # Store future in data for access later
@@ -314,7 +350,7 @@ def sherlock(username, site_data, query_notify,
         except:
             http_status = "?"
         try:
-            response_text = r.text.encode(r.encoding)
+            response_text = r.text.encode(r.encoding or "UTF-8")
         except:
             response_text = ""
 

sites.md

@@ -8,6 +8,7 @@
 1. [Airliners](https://www.airliners.net/)
 1. [Alik.cz](https://www.alik.cz/)
 1. [AllMyLinks](https://allmylinks.com/)
+1. [Anilist](https://anilist.co/)
 1. [Apple Developer](https://developer.apple.com)
 1. [Apple Discussions](https://discussions.apple.com)
 1. [Archive.org](https://archive.org)