diff --git a/.gitignore b/.gitignore index 119edc5d..3bf7f81b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,5 +22,8 @@ src/ # Comma-Separated Values (CSV) Reports *.csv +# Excluded sites list +tests/.excluded_sites + # MacOS Folder Metadata File .DS_Store diff --git a/README.md b/README.md index 6aaee5ab..cd8bea22 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,10 @@ Note that we do currently have 100% test coverage. Unfortunately, some of the sites that Sherlock checks are not always reliable, so it is common to get response errors. +If some sites are failing due to conection problems (site is down, in maintainence, etc) +you can exclude them from tests by creating a `tests/.excluded_sites` file with a +list of sites to ignore (one site name per line). + ## Stargazers over time [![Stargazers over time](https://starcharts.herokuapp.com/TheYahya/sherlock.svg)](https://starcharts.herokuapp.com/TheYahya/sherlock) diff --git a/data.json b/data.json index a51fe077..3d29318b 100644 --- a/data.json +++ b/data.json @@ -557,7 +557,7 @@ "url": "https://flipboard.com/@{}", "urlMain": "https://flipboard.com/", "username_claimed": "blue", - "username_unclaimed": "xgtrq" + "username_unclaimed": "noonewould" }, "FortniteTracker": { "errorType": "status_code", diff --git a/sherlock.py b/sherlock.py index 9c8af67a..d329789c 100755 --- a/sherlock.py +++ b/sherlock.py @@ -30,9 +30,10 @@ __version__ = "0.9.12" amount = 0 -global proxy_list +global proxy_list, timeout_limit proxy_list = [] +timeout_limit = 10 class ElapsedFuturesSession(FuturesSession): """ @@ -246,11 +247,13 @@ def sherlock(username, site_data, verbose=False, tor=False, unique_tor=False, pr proxies = {"http": proxy, "https": proxy} future = request_method(url=url_probe, headers=headers, proxies=proxies, - allow_redirects=allow_redirects + allow_redirects=allow_redirects, + timeout=timeout_limit ) else: future = request_method(url=url_probe, headers=headers, - allow_redirects=allow_redirects + allow_redirects=allow_redirects, + timeout=timeout_limit ) # Store future in data for access later diff --git a/tests/base.py b/tests/base.py index ff4c5416..4f1bee75 100644 --- a/tests/base.py +++ b/tests/base.py @@ -32,6 +32,14 @@ class SherlockBaseTest(unittest.TestCase): with open(data_file_path, "r", encoding="utf-8") as raw: self.site_data_all = json.load(raw) + # Load excluded sites list, if any + excluded_sites_path = os.path.join(os.path.dirname(os.path.realpath(sherlock.__file__)), "tests/.excluded_sites") + try: + with open(excluded_sites_path, "r", encoding="utf-8") as excluded_sites_file: + self.excluded_sites = excluded_sites_file.read().splitlines() + except FileNotFoundError: + self.excluded_sites = [] + self.verbose=False self.tor=False self.unique_tor=False @@ -134,6 +142,7 @@ class SherlockBaseTest(unittest.TestCase): for site, site_data in self.site_data_all.items(): if ( + (site in self.excluded_sites) or (site_data["errorType"] != detect_type) or (site_data.get("username_claimed") is None) or (site_data.get("username_unclaimed") is None)