From 601b72c165ae374af91bb1731505b2a65f980eb6 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Mon, 8 Apr 2024 18:57:34 -0400 Subject: [PATCH 1/4] Add multiprocessing control Argument --max-workers is added to Sherlock, default 20, allowing the user to specify the maximum number of workers. If the sitelist is lower than the maximum, it will not create more workers than sites. --- sherlock/sherlock.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sherlock/sherlock.py b/sherlock/sherlock.py index fb9f524..3beb2d8 100644 --- a/sherlock/sherlock.py +++ b/sherlock/sherlock.py @@ -163,6 +163,7 @@ def sherlock( unique_tor=False, proxy=None, timeout=60, + max_workers:int=20 ): """Run Sherlock Analysis. @@ -209,9 +210,7 @@ def sherlock( # Limit number of workers to 20. # This is probably vastly overkill. - if len(site_data) >= 20: - max_workers = 20 - else: + if len(site_data) < max_workers: max_workers = len(site_data) # Create multi-threaded session for all requests. @@ -643,6 +642,14 @@ def main(): help="Include checking of NSFW sites from default list.", ) + parser.add_argument( + "--max-workers", + type=int, + default=20, + dest="max_workers", + help="Set the maximum number of workers for Sherlock (Default: 20)" + ) + args = parser.parse_args() # If the user presses CTRL-C, exit gracefully without throwing errors @@ -764,6 +771,7 @@ def main(): unique_tor=args.unique_tor, proxy=args.proxy, timeout=args.timeout, + max_workers=args.max_workers ) if args.output: From 4cedcabacafffe7568180b1a6df587d8fc6fac45 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Mon, 8 Apr 2024 21:57:41 -0400 Subject: [PATCH 2/4] Add --max-workers to readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f8b1c02..e05795f 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,8 @@ optional arguments: --browse, -b Browse to all results on default browser. --local, -l Force the use of the local data.json file. --nsfw Include checking of NSFW sites from default list. + --max-workers MAX_WORKERS + Set the maximum number of workers for Sherlock (Default: 20) ``` To search for only one user: From 7c2c2ab82aac0cdc6e145b5729d838cc123c45ac Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Tue, 9 Apr 2024 22:07:45 -0400 Subject: [PATCH 3/4] Update workerctl implementation --- README.md | 7 +++---- sherlock/sherlock.py | 17 ++++++++--------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index e05795f..3d23d3a 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,8 @@ $ python3 sherlock --help usage: sherlock [-h] [--version] [--verbose] [--folderoutput FOLDEROUTPUT] [--output OUTPUT] [--tor] [--unique-tor] [--csv] [--xlsx] [--site SITE_NAME] [--proxy PROXY_URL] [--json JSON_FILE] - [--timeout TIMEOUT] [--print-all] [--print-found] [--no-color] - [--browse] [--local] [--nsfw] + [--workers WORKERS] [--timeout TIMEOUT] [--print-all] + [--print-found] [--no-color] [--browse] [--local] [--nsfw] USERNAMES [USERNAMES ...] Sherlock: Find Usernames Across Social Networks (Version 0.14.3) @@ -77,6 +77,7 @@ optional arguments: Make requests over a proxy. e.g. socks5://127.0.0.1:1080 --json JSON_FILE, -j JSON_FILE Load data from a JSON file or an online, valid, JSON file. + --workers WORKERS Set the maximum number of workers (Default: 20) --timeout TIMEOUT Time (in seconds) to wait for response to requests (Default: 60) --print-all Output sites where the username was not found. --print-found Output sites where the username was found. @@ -84,8 +85,6 @@ optional arguments: --browse, -b Browse to all results on default browser. --local, -l Force the use of the local data.json file. --nsfw Include checking of NSFW sites from default list. - --max-workers MAX_WORKERS - Set the maximum number of workers for Sherlock (Default: 20) ``` To search for only one user: diff --git a/sherlock/sherlock.py b/sherlock/sherlock.py index 3beb2d8..3941395 100644 --- a/sherlock/sherlock.py +++ b/sherlock/sherlock.py @@ -581,6 +581,13 @@ def main(): default=None, help="Load data from a JSON file or an online, valid, JSON file.", ) + parser.add_argument( + "--workers", + type=int, + default=20, + dest="workers", + help="Set the maximum number of workers for Sherlock (Default: 20)" + ) parser.add_argument( "--timeout", action="store", @@ -642,14 +649,6 @@ def main(): help="Include checking of NSFW sites from default list.", ) - parser.add_argument( - "--max-workers", - type=int, - default=20, - dest="max_workers", - help="Set the maximum number of workers for Sherlock (Default: 20)" - ) - args = parser.parse_args() # If the user presses CTRL-C, exit gracefully without throwing errors @@ -771,7 +770,7 @@ def main(): unique_tor=args.unique_tor, proxy=args.proxy, timeout=args.timeout, - max_workers=args.max_workers + max_workers=args.workers ) if args.output: From a85f7c4238617ac1f14a70b492943372774126f8 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Tue, 23 Apr 2024 00:46:46 -0400 Subject: [PATCH 4/4] Fix docu --- sherlock/sherlock.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sherlock/sherlock.py b/sherlock/sherlock.py index 3941395..e01772b 100644 --- a/sherlock/sherlock.py +++ b/sherlock/sherlock.py @@ -208,8 +208,7 @@ def sherlock( underlying_session = requests.session() underlying_request = requests.Request() - # Limit number of workers to 20. - # This is probably vastly overkill. + # Reduce worker count if greater than size of target pool if len(site_data) < max_workers: max_workers = len(site_data)