Merge branch 'master' into master

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -19,6 +19,15 @@ body:
         - Other (indicate below)
     validations:
       required: true
+  - type: input
+    id: package-version
+    attributes:
+      label: Package version
+      description: |
+        Knowing the version of the package you are using can help us diagnose your issue more quickly.
+        You can find the version by running `sherlock --version`.
+    validations:
+      required: true
   - type: textarea
     id: description
     attributes:

.github/workflows/regression.yml

@@ -49,7 +49,6 @@ jobs:
           macos-latest,
         ]
         python-version: [
-          '3.8',
           '3.9',
           '3.10',
           '3.11',

docs/README.md

@@ -22,18 +22,21 @@
 
 ## Installation
 
+> [!WARNING]  
+> Packages for ParrotOS and Ubuntu 24.04, maintained by a third party, appear to be __broken__.  
+> Users of these systems should defer to pipx/pip or Docker.
 
-| | Command | Notes |
+| Method | Notes |
-| - | - | - |
+| - | - |
-| PyPI | `pipx install sherlock-project` | `pip` may be used in place of `pipx` |
+| `pipx install sherlock-project` | `pip` may be used in place of `pipx` |
-| Docker | `docker pull sherlock/sherlock` | |
+| `docker run -it --rm sherlock/sherlock` |
-| Debian family | `apt install sherlock` | Kali, Parrot, Debian Testing and Sid |
+| `dnf install sherlock-project` | |
-| BlackArch | `pacman -S sherlock` |  |
+
-| Homebrew | `brew install sherlock` | |
+Community-maintained packages are available for Debian (>= 13), Ubuntu (>= 22.10), Homebrew, Kali, and BlackArch. These packages are not directly supported or maintained by the Sherlock Project.
 
 See all alternative installation methods [here](https://sherlockproject.xyz/installation)
 
-## Usage
+## General usage
 
 To search for only one user:
 ```bash

pyproject.toml

@@ -40,7 +40,7 @@ repository = "https://github.com/sherlock-project/sherlock"
 "Bug Tracker" = "https://github.com/sherlock-project/sherlock/issues"
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.9"
 certifi = ">=2019.6.16"
 colorama = "^0.4.1"
 PySocks = "^1.7.0"
@@ -48,8 +48,7 @@ requests = "^2.22.0"
 requests-futures = "^1.0.0"
 stem = "^1.8.0"
 torrequest = "^0.1.0"
-# pandas can likely be bumped up to ^2.0.0 after fc39 EOL
+pandas = "^2.2.1"
-pandas = ">=1.0.0,<3.0.0"
 openpyxl = "^3.0.10"
 
 [tool.poetry.extras]

sherlock_project/__main__.py

@@ -14,8 +14,8 @@ if __name__ == "__main__":
     # Check if the user is using the correct version of Python
     python_version = sys.version.split()[0]
 
-    if sys.version_info < (3, 8):
+    if sys.version_info < (3, 9):
-        print(f"Sherlock requires Python 3.8+\nYou are using Python {python_version}, which is not supported by Sherlock.")
+        print(f"Sherlock requires Python 3.9+\nYou are using Python {python_version}, which is not supported by Sherlock.")
         sys.exit(1)
 
     from sherlock_project import sherlock

sherlock_project/resources/data.json

@@ -31,9 +31,13 @@
     "username_claimed": "blue"
   },
   "8tracks": {
-    "errorMsg": "This page has vanished",
     "errorType": "message",
+    "errorMsg": "\"available\":true",
+    "headers": {
+      "Accept-Language": "en-US,en;q=0.5"
+    },
     "url": "https://8tracks.com/{}",
+    "urlProbe": "https://8tracks.com/users/check_username?login={}&format=jsonh",
     "urlMain": "https://8tracks.com/",
     "username_claimed": "blue"
   },
@@ -159,7 +163,8 @@
     "__comment__": "'The resource could not be found' relates to archive downtime",
     "errorMsg": [
       "could not fetch an account with user item identifier",
-      "The resource could not be found"
+      "The resource could not be found",
+      "Internet Archive services are temporarily offline"
     ],
     "errorType": "message",
     "url": "https://archive.org/details/@{}",
@@ -193,6 +198,12 @@
     "urlMain": "https://ask.fm/",
     "username_claimed": "blue"
   },
+  "Atcoder": {
+    "errorType": "status_code",
+    "url": "https://atcoder.jp/users/{}",
+    "urlMain": "https://atcoder.jp/",
+    "username_claimed": "ksun48"
+  },
   "Audiojungle": {
     "errorType": "status_code",
     "regexCheck": "^[a-zA-Z0-9_]+$",
@@ -298,11 +309,12 @@
     "urlMain": "https://www.blogger.com/",
     "username_claimed": "blue"
   },
-  "BodyBuilding": {
+  "BoardGameGeek": {
-    "errorType": "response_url",
+    "errorType": "message",
-    "errorUrl": "https://bodyspace.bodybuilding.com/",
+    "regexCheck": "^[a-zA-Z0-9_]*$",
-    "url": "https://bodyspace.bodybuilding.com/{}",
+    "errorMsg": "User not found",
-    "urlMain": "https://bodyspace.bodybuilding.com/",
+    "url": "https://boardgamegeek.com/user/{}",
+    "urlMain": "https://boardgamegeek.com",
     "username_claimed": "blue"
   },
   "BongaCams": {
@@ -695,6 +707,12 @@
     "urlMain": "https://community.eintracht.de/",
     "username_claimed": "mmammu"
   },
+  "Empretienda AR": {
+    "errorType": "status_code",
+    "url": "https://{}.empretienda.com.ar",
+    "urlMain": "https://empretienda.com",
+    "username_claimed": "camalote"
+  },
   "Envato Forum": {
     "errorType": "status_code",
     "url": "https://forums.envato.com/u/{}",
@@ -714,6 +732,12 @@
     "urlMain": "https://exposure.co/",
     "username_claimed": "jonasjacobsson"
   },
+  "exophase": {
+    "errorType": "status_code",
+    "url": "https://www.exophase.com/user/{}/",
+    "urlMain": "https://www.exophase.com/",
+    "username_claimed": "blue"
+  },
   "EyeEm": {
     "errorType": "status_code",
     "url": "https://www.eyeem.com/u/{}",
@@ -1688,6 +1712,12 @@
     "urlMain": "https://www.rajce.idnes.cz/",
     "username_claimed": "blue"
   },
+  "Rarible": {
+    "errorType": "status_code",
+    "url": "https://rarible.com/marketplace/api/v4/urls/{}",
+    "urlMain": "https://rarible.com/",
+    "username_claimed": "blue"
+  },
   "Rate Your Music": {
     "errorType": "status_code",
     "url": "https://rateyourmusic.com/~{}",
@@ -1918,11 +1948,10 @@
     "username_claimed": "adam"
   },
   "Speedrun.com": {
-    "errorMsg": "Not found",
+    "errorType": "status_code",
-    "errorType": "message",
+    "url": "https://speedrun.com/users/{}",
-    "url": "https://speedrun.com/user/{}",
     "urlMain": "https://speedrun.com/",
-    "username_claimed": "3Tau"
+    "username_claimed": "example"
   },
   "Spells8": {
     "errorType": "status_code",
@@ -2011,6 +2040,12 @@
     "urlProbe": "https://ch.tetr.io/api/users/{}",
     "username_claimed": "osk"
   },
+  "Tiendanube": {
+    "url": "https://{}.mitiendanube.com/",
+    "urlMain": "https://www.tiendanube.com/",
+    "errorType": "status_code",
+    "username_claimed": "blue"
+  },
   "TLDR Legal": {
     "errorType": "status_code",
     "regexCheck": "^[a-zA-Z0-9]{3,20}$",
@@ -2018,6 +2053,14 @@
     "urlMain": "https://tldrlegal.com/",
     "username_claimed": "kevin"
   },
+  "Topcoder": {
+    "errorType": "status_code",
+    "url": "https://profiles.topcoder.com/{}/",
+    "urlMain": "https://topcoder.com/",
+    "username_claimed": "USER",
+    "urlProbe": "https://api.topcoder.com/v5/members/{}",
+    "regexCheck": "[a-zA-Z0-9 ]"
+  },
   "TRAKTRAIN": {
     "errorType": "status_code",
     "url": "https://traktrain.com/{}",
@@ -2139,7 +2182,7 @@
     "regexCheck": "^[a-zA-Z0-9_]{1,15}$",
     "url": "https://x.com/{}",
     "urlMain": "https://x.com/",
-    "urlProbe": "https://nitter.net/{}",
+    "urlProbe": "https://nitter.privacydev.net/{}",
     "username_claimed": "blue"
   },
   "Typeracer": {
@@ -2181,6 +2224,12 @@
     "urlMain": "https://vsco.co/",
     "username_claimed": "blue"
   },
+  "Velog": {
+    "errorType": "status_code",
+    "url": "https://velog.io/@{}/posts",
+    "urlMain": "https://velog.io/",
+    "username_claimed": "qlgks1"
+  },
   "Velomania": {
     "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u0437\u0430\u0440\u0435\u0433\u0438\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u043d \u0438 \u043d\u0435 \u0438\u043c\u0435\u0435\u0442 \u043f\u0440\u043e\u0444\u0438\u043b\u044f \u0434\u043b\u044f \u043f\u0440\u043e\u0441\u043c\u043e\u0442\u0440\u0430.",
     "errorType": "message",
@@ -2223,6 +2272,12 @@
     "urlProbe": "https://www.virustotal.com/ui/users/{}/avatar",
     "username_claimed": "blue"
   },
+  "VLR": {
+    "errorType": "status_code",
+    "url": "https://www.vlr.gg/user/{}",
+    "urlMain": "https://www.vlr.gg",
+    "username_claimed": "optms"
+  },
   "WICG Forum": {
     "errorType": "status_code",
     "regexCheck": "^(?![.-])[a-zA-Z0-9_.-]{3,20}$",

sherlock_project/sherlock.py

@@ -24,6 +24,7 @@ import re
 from argparse import ArgumentParser, RawDescriptionHelpFormatter
 from json import loads as json_loads
 from time import monotonic
+from typing import Optional
 
 import requests
 from requests_futures.sessions import FuturesSession
@@ -167,14 +168,14 @@ def multiple_usernames(username):
 
 
 def sherlock(
-    username,
+    username: str,
-    site_data,
+    site_data: dict,
     query_notify: QueryNotify,
     tor: bool = False,
     unique_tor: bool = False,
     dump_response: bool = False,
-    proxy=None,
+    proxy: Optional[str] = None,
-    timeout=60,
+    timeout: int = 60,
 ):
     """Run Sherlock Analysis.
 
@@ -474,7 +475,7 @@ def sherlock(
             raise ValueError(
                 f"Unknown Error Type '{error_type}' for " f"site '{social_network}'"
             )
-        
+
         if dump_response:
             print("+++++++++++++++++++++")
             print(f"TARGET NAME   : {social_network}")
@@ -784,7 +785,24 @@ def main():
                 os.path.join(os.path.dirname(__file__), "resources/data.json")
             )
         else:
-            sites = SitesInformation(args.json_file)
+            json_file_location = args.json_file
+            if args.json_file:
+                # If --json parameter is a number, interpret it as a pull request number
+                if args.json_file.isnumeric():
+                    pull_number = args.json_file
+                    pull_url = f"https://api.github.com/repos/sherlock-project/sherlock/pulls/{pull_number}"
+                    pull_request_raw = requests.get(pull_url).text
+                    pull_request_json = json_loads(pull_request_raw)
+
+                    # Check if it's a valid pull request
+                    if "message" in pull_request_json:
+                        print(f"ERROR: Pull request #{pull_number} not found.")
+                        sys.exit(1)
+
+                    head_commit_sha = pull_request_json["head"]["sha"]
+                    json_file_location = f"https://raw.githubusercontent.com/sherlock-project/sherlock/{head_commit_sha}/sherlock_project/resources/data.json"
+
+            sites = SitesInformation(json_file_location)
     except Exception as error:
         print(f"ERROR:  {error}")
         sys.exit(1)

tests/sherlock_interactives.py

@@ -7,8 +7,8 @@ class Interactives:
     def run_cli(args:str = "") -> str:
         """Pass arguments to Sherlock as a normal user on the command line"""
         # Adapt for platform differences (Windows likes to be special)
-        if platform.system == "Windows":
+        if platform.system() == "Windows":
-            command:str = f"py -m sherlock {args}"
+            command:str = f"py -m sherlock_project {args}"
         else:
             command:str = f"sherlock {args}"
 
@@ -20,8 +20,7 @@ class Interactives:
             raise InteractivesSubprocessError(e.output.decode())
 
 
-    # -> list[str] is prefered, but will require deprecation of support for Python 3.8
+    def walk_sherlock_for_files_with(pattern: str) -> list[str]:
-    def walk_sherlock_for_files_with(pattern: str) -> list:
         """Check all files within the Sherlock package for matching patterns"""
         pattern:re.Pattern = re.compile(pattern)
         matching_files:list[str] = []

tests/test_probes.py

@@ -44,7 +44,7 @@ class TestLiveTargets:
 
     # Known positives should only use sites trusted to be reliable and unchanging
     @pytest.mark.parametrize('site,username',[
-        ('BodyBuilding', 'blue'),
+        ('Keybase', 'blue'),
         ('devRant', 'blue'),
     ])
     def test_known_positives_via_response_url(self, sites_info, site, username):

tox.ini

@@ -41,4 +41,3 @@ python =
     3.11: py311
     3.10: py310
     3.9: py39
-    3.8: py38