-
Notifications
You must be signed in to change notification settings - Fork 0
/
checker.py
52 lines (45 loc) · 1.71 KB
/
checker.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
import requests
from concurrent.futures import ThreadPoolExecutor, as_completed
import time
MAX_CONCURRENT_REQUESTS = 5
TIMEOUT = 10 # Adjust timeout as needed
def check_redirect(url):
try:
response = requests.get(url, allow_redirects=True, timeout=TIMEOUT)
final_url = response.url
if "login" not in final_url:
return final_url
else:
return None
except requests.RequestException as e:
print(f"Error checking {url}: {str(e)}")
return None
except requests.Timeout:
print(f"Timeout checking {url} after {TIMEOUT} seconds.")
return None
def main():
input_file = "clean_sites.txt"
output_file = "output.txt"
with open(input_file, "r") as infile:
sites = [line.strip() for line in infile]
with ThreadPoolExecutor(max_workers=MAX_CONCURRENT_REQUESTS) as executor:
futures = {
executor.submit(check_redirect, site): site
for site in sites
}
for future in as_completed(futures):
site = futures[future]
try:
redirected_url = future.result()
if redirected_url:
print(f"Redirected: {site} -> {redirected_url}")
with open(output_file, "a") as outfile:
outfile.write(f"{site} -> {redirected_url}\n")
else:
print(f"Skipped: {site} (contains 'login')")
except Exception as e:
print(f"Error processing {site}: {str(e)}")
if __name__ == "__main__":
start_time = time.time()
main()
print(f"Execution time: {time.time() - start_time} seconds")