-
Notifications
You must be signed in to change notification settings - Fork 2
/
mcd-bs4-lk.py
160 lines (125 loc) · 5.93 KB
/
mcd-bs4-lk.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
import requests as r
from bs4 import BeautifulSoup as BS
import pandas as pd
import datetime as dt
import pytz
import re
from pathlib import Path # install pathlib2 instead of pathlib
import traceback
# Reflects local time (SLST)
local_datetime = dt.datetime.now(pytz.timezone("Asia/Colombo"))
# Set headers to make HTTP request to seem to be from a normal browser
my_headers = {
"Access-Control-Allow-Origin": "*",
"Access-Control-Allow-Methods": "GET",
"Access-Control-Allow-Headers": "Content-Type",
"Access-Control-Max-Age": "3600",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml; q=0.9,image/webp,image/apng,*/*;q=0.8"
}
session = r.Session()
try:
# --------------------------------------- #
# Getting the Live Exchange Rate from XE #
# --------------------------------------- #
# Getting the correct XE webpage (all elements)
XE = BS(session.get("https://www.xe.com/currencyconverter/convert/?Amount=1&From=LKR&To=USD",
headers=my_headers).content, "lxml")
# Scraping the text from the selected element (CSS selector)
# Extracting only the number from the text string and converting it to a float value (decimal number)
# findall() and select() methods return a list, indicate index [0] to extract the first element as a string value
exchange_rate = float(
re.findall(
r"[-+]?(?:\d*\.\d+|\d+)",
XE.select("span[class*=faded-digits]")[0].parent.text
)[0]
)
print(
f"1 LKR = {exchange_rate} USD (1 USD = {1/exchange_rate} LKR) on {local_datetime.strftime('%A, %-d %B %Y')}"
)
# --------------------------------------- #
# List of URLS of Webpages to be Scraped #
# --------------------------------------- #
start_URLs = [
"https://www.mcdelivery.lk/lk/browse/menu.html?daypartId=1&catId=2", # Regular Menu
"https://www.mcdelivery.lk/lk/browse/menu.html?daypartId=2&catId=2" # Breakfast Menu
]
# Populated by scraping href values of anchor tags
URL_list = []
# -------------------------------------- #
# Parsing the data into Dictionary List #
# -------------------------------------- #
# Initialising the list object [] to hold dictionaries {}
product_list = []
# Outer For Loop iterates through first category page of each menu type to get links to subsequent categories
for url in start_URLs:
first_page = BS(session.get(url, headers=my_headers).content, "lxml")
links = ("https://www.mcdelivery.lk/lk/browse/menu.html" + a["href"] for a in (
first_page.select("li.secondary-menu-item:not([class*='selected']) a[href]")))
URL_list.extend(links)
# Inner For Loop scrapes the menu data from first category page
for products in first_page.select("div.product-card"):
product = {}
product["Date"] = local_datetime.strftime("%Y/%m/%d")
product["Day"] = local_datetime.strftime("%a")
product["Territory"] = "Sri Lanka"
product["Menu Item"] = products.select("h5.product-title")[0].text
# Regex for currencies in large denominations with "," thousands separator
product["Price (LKR)"] = float((re.findall(r"[-+]?(?:\d*\,\d+|\d+)",
products.select("span.starting-price")[0].text)[0]).replace(",", ""))
product["Price (USD)"] = round(
(product["Price (LKR)"] * exchange_rate), 2)
product["Category"] = first_page.select(
"ol.breadcrumb > li.active")[0].text
product["Menu"] = first_page.select(
"li.primary-menu-item.selected > a > span")[0].text
product_list.append(product)
# Second Outer For Loop iterates through list of generated URLs
for url in URL_list:
next_page = BS(session.get(url, headers=my_headers).content, "lxml")
# Second Inner For Loop iterates through elements on all other pages
for products in next_page.select("div.product-card"):
product = {}
product["Date"] = local_datetime.strftime("%Y/%m/%d")
product["Day"] = local_datetime.strftime("%a")
product["Territory"] = "Sri Lanka"
product["Menu Item"] = products.select("h5.product-title")[0].text
# Regex for currencies in large denominations with "," thousands separator
product["Price (LKR)"] = float((re.findall(r"[-+]?(?:\d*\,\d+|\d+)",
products.select("span.starting-price")[0].text)[0]).replace(",", ""))
product["Price (USD)"] = round(
(product["Price (LKR)"] * exchange_rate), 2)
product["Category"] = next_page.select(
"ol.breadcrumb > li.active")[0].text
product["Menu"] = next_page.select(
"li.primary-menu-item.selected > a > span")[0].text
product_list.append(product)
# ---------------------------------------------------- #
# Constructing the Dataframe and Exporting it to File #
# ---------------------------------------------------- #
product_list_df = pd.DataFrame(product_list)
product_list_df.drop_duplicates(
subset=None, keep='last', inplace=True, ignore_index=True)
product_list_df.reset_index(drop=True, inplace=True)
product_list_df.index = pd.RangeIndex(
start=1, stop=(len(product_list_df.index) + 1), step=1)
print(product_list_df)
timestamp = str(local_datetime.strftime("[%Y-%m-%d %H:%M:%S]"))
output_file = str(timestamp + " mcd-bs4-lk.csv")
output_dir = Path("./scraped-data")
# Create directory as required; won't raise an error if directory already exists
output_dir.mkdir(parents=True, exist_ok=True)
product_list_df.to_csv((output_dir / output_file),
float_format="%.2f", encoding="utf-8")
# Output filename format: "[YYYY-MM-DD hh:mm:ss] mcd-bs4-lk.csv"
except Exception:
print(
f'''
\n\n
---
One or more errors occurred:
{traceback.format_exc()}
---
\n\n
'''
)