Merge branch 'master' into master

3 weeks ago · 6f3b89c98a
parent 0b7d925b50 785346c12d
commit 6f3b89c98a
11 changed files with 119 additions and 38 deletions
--- a/.github/ISSUE_TEMPLATE/bug-report.yml
+++ b/.github/ISSUE_TEMPLATE/bug-report.yml
@ -19,6 +19,15 @@ body:
        - Other (indicate below)
    validations:
      required: true
+  - type: input
+    id: package-version
+    attributes:
+      label: Package version
+      description: |
+        Knowing the version of the package you are using can help us diagnose your issue more quickly.
+        You can find the version by running `sherlock --version`.
+    validations:
+      required: true
  - type: textarea
    id: description
    attributes:
--- a/.github/workflows/regression.yml
+++ b/.github/workflows/regression.yml
@ -49,7 +49,6 @@ jobs:
          macos-latest,
        ]
        python-version: [
-          '3.8',
          '3.9',
          '3.10',
          '3.11',
--- a/docs/README.md
+++ b/docs/README.md
@ -22,18 +22,21 @@

 ## Installation

+> [!WARNING]  
+> Packages for ParrotOS and Ubuntu 24.04, maintained by a third party, appear to be __broken__.  
+> Users of these systems should defer to pipx/pip or Docker.

-| | Command | Notes |
-| - | - | - |
-| PyPI | `pipx install sherlock-project` | `pip` may be used in place of `pipx` |
-| Docker | `docker pull sherlock/sherlock` | |
-| Debian family | `apt install sherlock` | Kali, Parrot, Debian Testing and Sid |
-| BlackArch | `pacman -S sherlock` |  |
-| Homebrew | `brew install sherlock` | |
+| Method | Notes |
+| - | - |
+| `pipx install sherlock-project` | `pip` may be used in place of `pipx` |
+| `docker run -it --rm sherlock/sherlock` |
+| `dnf install sherlock-project` | |
+
+Community-maintained packages are available for Debian (>= 13), Ubuntu (>= 22.10), Homebrew, Kali, and BlackArch. These packages are not directly supported or maintained by the Sherlock Project.

 See all alternative installation methods [here](https://sherlockproject.xyz/installation)

-## Usage
+## General usage

 To search for only one user:
 ```bash
--- a/pyproject.toml
+++ b/pyproject.toml
@ -40,7 +40,7 @@ repository = "https://github.com/sherlock-project/sherlock"
 "Bug Tracker" = "https://github.com/sherlock-project/sherlock/issues"

 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.9"
 certifi = ">=2019.6.16"
 colorama = "^0.4.1"
 PySocks = "^1.7.0"
@ -48,8 +48,7 @@ requests = "^2.22.0"
 requests-futures = "^1.0.0"
 stem = "^1.8.0"
 torrequest = "^0.1.0"
-# pandas can likely be bumped up to ^2.0.0 after fc39 EOL
-pandas = ">=1.0.0,<3.0.0"
+pandas = "^2.2.1"
 openpyxl = "^3.0.10"

 [tool.poetry.extras]
--- a/sherlock_project/main.py
+++ b/sherlock_project/main.py
@ -14,8 +14,8 @@ if __name__ == "__main__":
    # Check if the user is using the correct version of Python
    python_version = sys.version.split()[0]

-    if sys.version_info < (3, 8):
-        print(f"Sherlock requires Python 3.8+\nYou are using Python {python_version}, which is not supported by Sherlock.")
+    if sys.version_info < (3, 9):
+        print(f"Sherlock requires Python 3.9+\nYou are using Python {python_version}, which is not supported by Sherlock.")
        sys.exit(1)

    from sherlock_project import sherlock
--- a/sherlock_project/py.typed
+++ b/sherlock_project/py.typed
--- a/sherlock_project/resources/data.json
+++ b/sherlock_project/resources/data.json
@ -31,9 +31,13 @@
    "username_claimed": "blue"
  },
  "8tracks": {
-    "errorMsg": "This page has vanished",
    "errorType": "message",
+    "errorMsg": "\"available\":true",
+    "headers": {
+      "Accept-Language": "en-US,en;q=0.5"
+    },
    "url": "https://8tracks.com/{}",
+    "urlProbe": "https://8tracks.com/users/check_username?login={}&format=jsonh",
    "urlMain": "https://8tracks.com/",
    "username_claimed": "blue"
  },
@ -159,7 +163,8 @@
    "__comment__": "'The resource could not be found' relates to archive downtime",
    "errorMsg": [
      "could not fetch an account with user item identifier",
-      "The resource could not be found"
+      "The resource could not be found",
+      "Internet Archive services are temporarily offline"
    ],
    "errorType": "message",
    "url": "https://archive.org/details/@{}",
@ -193,6 +198,12 @@
    "urlMain": "https://ask.fm/",
    "username_claimed": "blue"
  },
+  "Atcoder": {
+    "errorType": "status_code",
+    "url": "https://atcoder.jp/users/{}",
+    "urlMain": "https://atcoder.jp/",
+    "username_claimed": "ksun48"
+  },
  "Audiojungle": {
    "errorType": "status_code",
    "regexCheck": "^[a-zA-Z0-9_]+$",
@ -298,11 +309,12 @@
    "urlMain": "https://www.blogger.com/",
    "username_claimed": "blue"
  },
-  "BodyBuilding": {
-    "errorType": "response_url",
-    "errorUrl": "https://bodyspace.bodybuilding.com/",
-    "url": "https://bodyspace.bodybuilding.com/{}",
-    "urlMain": "https://bodyspace.bodybuilding.com/",
+  "BoardGameGeek": {
+    "errorType": "message",
+    "regexCheck": "^[a-zA-Z0-9_]*$",
+    "errorMsg": "User not found",
+    "url": "https://boardgamegeek.com/user/{}",
+    "urlMain": "https://boardgamegeek.com",
    "username_claimed": "blue"
  },
  "BongaCams": {
@ -695,6 +707,12 @@
    "urlMain": "https://community.eintracht.de/",
    "username_claimed": "mmammu"
  },
+  "Empretienda AR": {
+    "errorType": "status_code",
+    "url": "https://{}.empretienda.com.ar",
+    "urlMain": "https://empretienda.com",
+    "username_claimed": "camalote"
+  },
  "Envato Forum": {
    "errorType": "status_code",
    "url": "https://forums.envato.com/u/{}",
@ -714,6 +732,12 @@
    "urlMain": "https://exposure.co/",
    "username_claimed": "jonasjacobsson"
  },
+  "exophase": {
+    "errorType": "status_code",
+    "url": "https://www.exophase.com/user/{}/",
+    "urlMain": "https://www.exophase.com/",
+    "username_claimed": "blue"
+  },
  "EyeEm": {
    "errorType": "status_code",
    "url": "https://www.eyeem.com/u/{}",
@ -1688,6 +1712,12 @@
    "urlMain": "https://www.rajce.idnes.cz/",
    "username_claimed": "blue"
  },
+  "Rarible": {
+    "errorType": "status_code",
+    "url": "https://rarible.com/marketplace/api/v4/urls/{}",
+    "urlMain": "https://rarible.com/",
+    "username_claimed": "blue"
+  },
  "Rate Your Music": {
    "errorType": "status_code",
    "url": "https://rateyourmusic.com/~{}",
@ -1918,11 +1948,10 @@
    "username_claimed": "adam"
  },
  "Speedrun.com": {
-    "errorMsg": "Not found",
-    "errorType": "message",
-    "url": "https://speedrun.com/user/{}",
+    "errorType": "status_code",
+    "url": "https://speedrun.com/users/{}",
    "urlMain": "https://speedrun.com/",
-    "username_claimed": "3Tau"
+    "username_claimed": "example"
  },
  "Spells8": {
    "errorType": "status_code",
@ -2011,6 +2040,12 @@
    "urlProbe": "https://ch.tetr.io/api/users/{}",
    "username_claimed": "osk"
  },
+  "Tiendanube": {
+    "url": "https://{}.mitiendanube.com/",
+    "urlMain": "https://www.tiendanube.com/",
+    "errorType": "status_code",
+    "username_claimed": "blue"
+  },
  "TLDR Legal": {
    "errorType": "status_code",
    "regexCheck": "^[a-zA-Z0-9]{3,20}$",
@ -2018,6 +2053,14 @@
    "urlMain": "https://tldrlegal.com/",
    "username_claimed": "kevin"
  },
+  "Topcoder": {
+    "errorType": "status_code",
+    "url": "https://profiles.topcoder.com/{}/",
+    "urlMain": "https://topcoder.com/",
+    "username_claimed": "USER",
+    "urlProbe": "https://api.topcoder.com/v5/members/{}",
+    "regexCheck": "[a-zA-Z0-9 ]"
+  },
  "TRAKTRAIN": {
    "errorType": "status_code",
    "url": "https://traktrain.com/{}",
@ -2139,7 +2182,7 @@
    "regexCheck": "^[a-zA-Z0-9_]{1,15}$",
    "url": "https://x.com/{}",
    "urlMain": "https://x.com/",
-    "urlProbe": "https://nitter.net/{}",
+    "urlProbe": "https://nitter.privacydev.net/{}",
    "username_claimed": "blue"
  },
  "Typeracer": {
@ -2181,6 +2224,12 @@
    "urlMain": "https://vsco.co/",
    "username_claimed": "blue"
  },
+  "Velog": {
+    "errorType": "status_code",
+    "url": "https://velog.io/@{}/posts",
+    "urlMain": "https://velog.io/",
+    "username_claimed": "qlgks1"
+  },
  "Velomania": {
    "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u0437\u0430\u0440\u0435\u0433\u0438\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u043d \u0438 \u043d\u0435 \u0438\u043c\u0435\u0435\u0442 \u043f\u0440\u043e\u0444\u0438\u043b\u044f \u0434\u043b\u044f \u043f\u0440\u043e\u0441\u043c\u043e\u0442\u0440\u0430.",
    "errorType": "message",
@ -2223,6 +2272,12 @@
    "urlProbe": "https://www.virustotal.com/ui/users/{}/avatar",
    "username_claimed": "blue"
  },
+  "VLR": {
+    "errorType": "status_code",
+    "url": "https://www.vlr.gg/user/{}",
+    "urlMain": "https://www.vlr.gg",
+    "username_claimed": "optms"
+  },
  "WICG Forum": {
    "errorType": "status_code",
    "regexCheck": "^(?![.-])[a-zA-Z0-9_.-]{3,20}$",
--- a/sherlock_project/sherlock.py
+++ b/sherlock_project/sherlock.py
@ -24,6 +24,7 @@ import re
 from argparse import ArgumentParser, RawDescriptionHelpFormatter
 from json import loads as json_loads
 from time import monotonic
+from typing import Optional

 import requests
 from requests_futures.sessions import FuturesSession
@ -167,14 +168,14 @@ def multiple_usernames(username):


 def sherlock(
-    username,
-    site_data,
+    username: str,
+    site_data: dict,
    query_notify: QueryNotify,
    tor: bool = False,
    unique_tor: bool = False,
    dump_response: bool = False,
-    proxy=None,
-    timeout=60,
+    proxy: Optional[str] = None,
+    timeout: int = 60,
 ):
    """Run Sherlock Analysis.

@ -784,7 +785,24 @@ def main():
                os.path.join(os.path.dirname(__file__), "resources/data.json")
            )
        else:
-            sites = SitesInformation(args.json_file)
+            json_file_location = args.json_file
+            if args.json_file:
+                # If --json parameter is a number, interpret it as a pull request number
+                if args.json_file.isnumeric():
+                    pull_number = args.json_file
+                    pull_url = f"https://api.github.com/repos/sherlock-project/sherlock/pulls/{pull_number}"
+                    pull_request_raw = requests.get(pull_url).text
+                    pull_request_json = json_loads(pull_request_raw)
+
+                    # Check if it's a valid pull request
+                    if "message" in pull_request_json:
+                        print(f"ERROR: Pull request #{pull_number} not found.")
+                        sys.exit(1)
+
+                    head_commit_sha = pull_request_json["head"]["sha"]
+                    json_file_location = f"https://raw.githubusercontent.com/sherlock-project/sherlock/{head_commit_sha}/sherlock_project/resources/data.json"
+
+            sites = SitesInformation(json_file_location)
    except Exception as error:
        print(f"ERROR:  {error}")
        sys.exit(1)
--- a/tests/sherlock_interactives.py
+++ b/tests/sherlock_interactives.py
@ -7,8 +7,8 @@ class Interactives:
    def run_cli(args:str = "") -> str:
        """Pass arguments to Sherlock as a normal user on the command line"""
        # Adapt for platform differences (Windows likes to be special)
-        if platform.system == "Windows":
-            command:str = f"py -m sherlock {args}"
+        if platform.system() == "Windows":
+            command:str = f"py -m sherlock_project {args}"
        else:
            command:str = f"sherlock {args}"

@ -20,8 +20,7 @@ class Interactives:
            raise InteractivesSubprocessError(e.output.decode())


-    # -> list[str] is prefered, but will require deprecation of support for Python 3.8
-    def walk_sherlock_for_files_with(pattern: str) -> list:
+    def walk_sherlock_for_files_with(pattern: str) -> list[str]:
        """Check all files within the Sherlock package for matching patterns"""
        pattern:re.Pattern = re.compile(pattern)
        matching_files:list[str] = []
--- a/tests/test_probes.py
+++ b/tests/test_probes.py
@ -44,7 +44,7 @@ class TestLiveTargets:

    # Known positives should only use sites trusted to be reliable and unchanging
    @pytest.mark.parametrize('site,username',[
-        ('BodyBuilding', 'blue'),
+        ('Keybase', 'blue'),
        ('devRant', 'blue'),
    ])
    def test_known_positives_via_response_url(self, sites_info, site, username):
--- a/tox.ini
+++ b/tox.ini
@ -41,4 +41,3 @@ python =
    3.11: py311
    3.10: py310
    3.9: py39
-    3.8: py38