-
Notifications
You must be signed in to change notification settings - Fork 0
/
index.py
110 lines (102 loc) · 5.07 KB
/
index.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
import requests as req
from bs4 import BeautifulSoup
import re
import csv
# This is a python script that scraps the Salatomatic website for all the Masjids in the USA
# This function scraps the main page of Salatomatic USA and gets the div that contains all the main state page
# Returns a list of all the masjids in the USA by state
# Format: [state#0, state#1, ...]
def getAllMasjidsinUSA():
allMasjidInformation = []
URL = "https://www.salatomatic.com/reg/United-States/sPpaNwWSpq"
r = req.get(URL)
soup = BeautifulSoup(r.content, 'html5lib')
sortedTableArray = []
#print(soup.prettify())
tableArray = soup.find_all('table')
for table in tableArray:
if(table.get_text().find("Alabama") != -1 and table.get_text().find("32") != -1):
#print(table.prettify())
print("FOUND")
sortedTableArray.append(table)
importantTable = sortedTableArray[(len(sortedTableArray))-1]
return getStateMasjidLinks(importantTable)
# This function sorts through the table from the getAllMasjidsinUSA() function and gets the links to each state
# Returns a list of all the masjids in a state, sorted by metro area
# Format: [state#0, state#1, ...]
def getStateMasjidLinks(importantTable):
# this table contains the links each State Page (which contains the metroplitan areas)
# Should be stored as [Name of the State, Link to the State]
stateLinks = []
for link in importantTable.find_all('a'):
#print(link.get('href'))
#print(link.string)
link_url = "https://www.salatomatic.com/reg/" + link.get('href')
stateLinks.extend(getMetropolitanMasjidLinks(link.string, link_url))
print(stateLinks)
return stateLinks
# This function sorts through each link from the getStateMasjidLinks() function and gets the links to each metropolitan area
# Returns a list of all the masjids in a metropolitan areas of the state
# [metropolitanArea#0, metropolitanArea#1, ...]
def getMetropolitanMasjidLinks(stateName, stateLink):
print(stateName)
URL = stateLink
r = req.get(URL)
soup = BeautifulSoup(r.content, 'html5lib')
# Format [metro name, link to metro]
metropolitanAreaLinks = []
for link in soup.find_all('a'):
if(link.get('href').find("/sub/") != -1):
#print(link.get('href'))
#print(link.string)
metropolitanAreaLinks.extend(getMasjidNamesAndLocations(stateName, link.string, "https://www.salatomatic.com" + link.get('href')))
print(metropolitanAreaLinks)
return metropolitanAreaLinks
#print(metropolitanAreaLinks)
#print(len(metropolitanAreaLinks))
# This function sorts through each link from the getMetropolitanMasjidLinks() function and gets the name and location of each masjid
# Returns a list of all the masjids in the metropolitan area
# Return: [masjid_info#0, masjid_info#1, ...]
def getMasjidNamesAndLocations(stateName, metropolitanAreaName, metropolitanAreaLink):
print(metropolitanAreaName)
URL = metropolitanAreaLink
r = req.get(URL)
soup = BeautifulSoup(r.content, 'html5lib')
# Format [salatomatic link, masjid name, location]
metroInfo = []
for div in soup.find_all('div', {'id': 'header', 'onclick': lambda value: value and 'location.href' in value}):
masjidName = div.find('div', class_= 'titleBS').a.string
masjidLink = "https://www.salatomatic.com" + div.find('div', class_= 'titleBS').a['href']
masjidLocation = div.find('div', class_= 'tinyLink').string
metroInfo.append(getMasjidPhoneNumber(stateName, metropolitanAreaName, masjidName, masjidLocation, masjidLink))
return metroInfo
#print(masjidLinksNamesAndLocations)
# This masjid gets the phone number of each masjid. The phone number may be None.
# Returns: {State Name, Metro Area, Masjid Name, Masjid Location, Phone Number}
def getMasjidPhoneNumber(stateName, metropolitanAreaName, masjidName, masjidLocation, masjidLink):
URL = masjidLink
r = req.get(URL)
soup = BeautifulSoup(r.content, 'html5lib')
# Check text of page to see if phone number exists
#print(soup.find_all('div', class_= 'midLink'))
phoneNumbers = re.findall(r"\(\d\d\d\)\s\d\d\d-\d\d\d\d", soup.get_text())
if(len(phoneNumbers) == 0):
phoneNumbers = None
else:
phoneNumbers = phoneNumbers[0]
masjid_data = {"State": stateName, "Metropolitan Area": metropolitanAreaName, "Masjid Name" : masjidName, "Masjid Location" : masjidLocation, "Phone Number" : phoneNumbers}
print(masjid_data)
return masjid_data
def save_to_csv(data, filename):
csv_file = filename
fieldnames = ["State", "Metropolitan Area", "Masjid Name", "Masjid Location", "Phone Number"]
with open(csv_file, mode="w",encoding='utf-8', errors='ignore', newline="") as file:
writer = csv.DictWriter(file, fieldnames=fieldnames)
# Write the header row
writer.writeheader()
# Write the data rows
writer.writerows(data)
print("CSV file created successfully.")
# TODO: Format into an excel sheet
allMasjidInfo = getAllMasjidsinUSA()
save_to_csv(allMasjidInfo, "masjid_data.csv")