From 737ce51a1500b300b4f8a6721a1cfc8231b7cc79 Mon Sep 17 00:00:00 2001 From: Seth Falco Date: Mon, 6 Dec 2021 19:08:49 +0100 Subject: [PATCH] feat: support post requests --- removed_sites.json | 8 ---- removed_sites.md | 14 ------ sherlock/resources/data.json | 20 ++++++++- sherlock/sherlock.py | 84 +++++++++++++++++++++++++----------- sites.md | 1 + 5 files changed, 79 insertions(+), 48 deletions(-) diff --git a/removed_sites.json b/removed_sites.json index a60886d7..a1399d2d 100644 --- a/removed_sites.json +++ b/removed_sites.json @@ -645,14 +645,6 @@ "username_claimed": "TheMorozko", "username_unclaimed": "noonewouldeverusethis7" }, - "Anilist": { - "errorType": "status_code", - "regexCheck": "^[A-Za-z0-9]{2,20}$", - "url": "https://anilist.co/user/{}/", - "urlMain": "https://anilist.co/", - "username_claimed": "Josh", - "username_unclaimed": "noonewouldeverusethi" - }, "Coil": { "errorMsg": "Whoops, the thing you were looking for isn't here", "errorType": "message", diff --git a/removed_sites.md b/removed_sites.md index 4ec050ff..7c13dd25 100644 --- a/removed_sites.md +++ b/removed_sites.md @@ -1254,20 +1254,6 @@ As of 2021-11-21, 1337x seems to be down causing false positives. } ``` -### Anilist -As of 2021-11-26, Anilist is returning false positives. - -``` - "Anilist": { - "errorType": "status_code", - "regexCheck": "^[A-Za-z0-9]{2,20}$", - "url": "https://anilist.co/user/{}/", - "urlMain": "https://anilist.co/", - "username_claimed": "Josh", - "username_unclaimed": "noonewouldeverusethi" - } -``` - ### Coil As of 2021-11-26, Coil is returning false positives. diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index 976b38d5..fc69e24d 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -65,6 +65,22 @@ "username_claimed": "blue", "username_unclaimed": "noonewouldeverusethis7" }, + "Anilist": { + "errorType": "status_code", + "regexCheck": "^[A-Za-z0-9]{2,20}$", + "request_method": "POST", + "request_payload": { + "query": "query($name:String){User(name:$name){id}}", + "variables": { + "name": "{}" + } + }, + "url": "https://anilist.co/user/{}/", + "urlProbe": "https://graphql.anilist.co/", + "urlMain": "https://anilist.co/", + "username_claimed": "Josh", + "username_unclaimed": "noonewouldeverusethi" + }, "Apple Developer": { "errorType": "status_code", "url": "https://developer.apple.com/forums/profile/{}", @@ -1792,7 +1808,7 @@ }, "Vero": { "errorType": "status_code", - "request_head_only": false, + "request_method": "GET", "url": "https://vero.co/{}", "urlMain": "https://vero.co/", "username_claimed": "blue", @@ -1815,7 +1831,7 @@ }, "VirusTotal": { "errorType": "status_code", - "request_head_only": false, + "request_method": "GET", "url": "https://www.virustotal.com/ui/users/{}/trusted_users", "urlMain": "https://www.virustotal.com/", "urlProbe": "https://www.virustotal.com/ui/users/{}/avatar", diff --git a/sherlock/sherlock.py b/sherlock/sherlock.py index 97beb77d..fa9f4d9f 100644 --- a/sherlock/sherlock.py +++ b/sherlock/sherlock.py @@ -125,6 +125,21 @@ def get_response(request_future, error_type, social_network): return response, error_context, expection_text +def interpolate_string(object, username): + """Insert a string into the string properties of an object recursively.""" + + if isinstance(object, str): + return object.replace("{}", username) + elif isinstance(object, dict): + for key, value in object.items(): + object[key] = interpolate_string(value, username) + elif isinstance(object, list): + for i in object: + object[i] = interpolate_string(object[i], username) + + return object + + def sherlock(username, site_data, query_notify, tor=False, unique_tor=False, proxy=None, timeout=None): @@ -207,7 +222,7 @@ def sherlock(username, site_data, query_notify, headers.update(net_info["headers"]) # URL of user on site (if it exists) - url = net_info["url"].format(username) + url = interpolate_string(net_info["url"], username) # Don't make request if username is invalid for the site regex_check = net_info.get("regexCheck") @@ -225,25 +240,44 @@ def sherlock(username, site_data, query_notify, # URL of user on site (if it exists) results_site["url_user"] = url url_probe = net_info.get("urlProbe") + request_method = net_info.get("request_method") + request_payload = net_info.get("request_payload") + request = None + + if request_method is not None: + if request_method == "GET": + request = session.get + elif request_method == "HEAD": + request = session.head + elif request_method == "POST": + request = session.post + elif request_method == "PUT": + request = session.put + else: + raise RuntimeError( f"Unsupported request_method for {url}") + + if request_payload is not None: + request_payload = interpolate_string(request_payload, username) + if url_probe is None: # Probe URL is normal one seen by people out on the web. url_probe = url else: # There is a special URL for probing existence separate # from where the user profile normally can be found. - url_probe = url_probe.format(username) - - if (net_info["errorType"] == 'status_code' and - net_info.get("request_head_only", True) == True): - # In most cases when we are detecting by status code, - # it is not necessary to get the entire body: we can - # detect fine with just the HEAD response. - request_method = session.head - else: - # Either this detect method needs the content associated - # with the GET response, or this specific website will - # not respond properly unless we request the whole page. - request_method = session.get + url_probe = interpolate_string(url_probe, username) + + if request is None: + if net_info["errorType"] == 'status_code': + # In most cases when we are detecting by status code, + # it is not necessary to get the entire body: we can + # detect fine with just the HEAD response. + request = session.head + else: + # Either this detect method needs the content associated + # with the GET response, or this specific website will + # not respond properly unless we request the whole page. + request = session.get if net_info["errorType"] == "response_url": # Site forwards request to a different URL if username not @@ -258,16 +292,18 @@ def sherlock(username, site_data, query_notify, # This future starts running the request in a new thread, doesn't block the main thread if proxy is not None: proxies = {"http": proxy, "https": proxy} - future = request_method(url=url_probe, headers=headers, - proxies=proxies, - allow_redirects=allow_redirects, - timeout=timeout - ) + future = request(url=url_probe, headers=headers, + proxies=proxies, + allow_redirects=allow_redirects, + timeout=timeout, + json=request_payload + ) else: - future = request_method(url=url_probe, headers=headers, - allow_redirects=allow_redirects, - timeout=timeout - ) + future = request(url=url_probe, headers=headers, + allow_redirects=allow_redirects, + timeout=timeout, + json=request_payload + ) # Store future in data for access later net_info["request_future"] = future @@ -314,7 +350,7 @@ def sherlock(username, site_data, query_notify, except: http_status = "?" try: - response_text = r.text.encode(r.encoding) + response_text = r.text.encode(r.encoding or "UTF-8") except: response_text = "" diff --git a/sites.md b/sites.md index 08d50cd6..f07eb5cc 100644 --- a/sites.md +++ b/sites.md @@ -8,6 +8,7 @@ 1. [Airliners](https://www.airliners.net/) 1. [Alik.cz](https://www.alik.cz/) 1. [AllMyLinks](https://allmylinks.com/) +1. [Anilist](https://anilist.co/) 1. [Apple Developer](https://developer.apple.com) 1. [Apple Discussions](https://discussions.apple.com) 1. [Archive.org](https://archive.org)