-
Notifications
You must be signed in to change notification settings - Fork 2
/
mcd-bs4-my.py
151 lines (118 loc) · 5.38 KB
/
mcd-bs4-my.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
import requests as r
from bs4 import BeautifulSoup as BS
import pandas as pd
import datetime as dt
import pytz
import re
import traceback
# Reflects UTC time with offset indicator
# local_datetime = pytz.timezone("Asia/Kuala_Lumpur").localize(dt.datetime.utcnow())
# Reflects local time
local_datetime = dt.datetime.now(pytz.timezone("Asia/Kuala_Lumpur"))
# Set headers to make HTTP request to seem to be from a normal browser
my_headers = {
"Access-Control-Allow-Origin": "*",
"Access-Control-Allow-Methods": "GET",
"Access-Control-Allow-Headers": "Content-Type",
"Access-Control-Max-Age": "3600",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml; q=0.9,image/webp,image/apng,*/*;q=0.8"
}
session = r.Session()
try:
# --------------------------------------- #
# Getting the Live Exchange Rate from XE #
# --------------------------------------- #
# Getting the correct XE webpage (all elements)
XE = BS(session.get("https://www.xe.com/currencyconverter/convert/?Amount=1&From=MYR&To=USD",
headers=my_headers).content, "lxml")
# Scraping the text from the selected element (CSS selector)
# Extracting only the number from the text string and converting it to a float value (decimal number)
# findall() and select() methods return a list, indicate index [0] to extract the first element as a string value
exchange_rate = float(
re.findall(
r"[-+]?(?:\d*\.\d+|\d+)",
XE.select("span[class*=faded-digits]")[0].parent.text
)[0]
)
print(
f"1 MYR = {exchange_rate} USD (1 USD = {1/exchange_rate} MYR) on {local_datetime.strftime('%A, %-d %B %Y')}"
)
# --------------------------------------- #
# List of URLS of Webpages to be Scraped #
# --------------------------------------- #
start_URLs = [
"https://www.mcdelivery.com.my/my/browse/menu.html?daypartId=9&catId=74&locale=en", # Regular Menu
# Breakfast Menu
"https://www.mcdelivery.com.my/my/browse/menu.html?daypartId=10&catId=74&locale=en"
]
# Populated by scraping href values of anchor tags
URL_list = []
# -------------------------------------- #
# Parsing the data into Dictionary List #
# -------------------------------------- #
# Initialising the list object [] to hold dictionaries {}
product_list = []
# Outer loop iterates through first category page of each menu type to get links to subsequent categories
for url in start_URLs:
first_page = BS(session.get(url, headers=my_headers).content, "lxml")
links = ("https://www.mcdelivery.com.my/my/browse/menu.html" + a["href"] + "&locale=en" for a in (
first_page.select("li.secondary-menu-item:not([class*='selected']) a[href]")))
URL_list.extend(links)
# Inner loop scrapes the menu data from first category page
for products in first_page.select("div.product-card"):
product = {}
product["Date"] = local_datetime.strftime("%Y/%m/%d")
product["Day"] = local_datetime.strftime("%a")
product["Territory"] = "Malaysia"
product["Menu Item"] = products.select("h5.product-title")[0].text
product["Price (MYR)"] = float((re.findall(
r"[-+]?(?:\d*\.\d+|\d+)", products.select("span.starting-price")[0].text)[0]))
product["Price (USD)"] = round(
(product["Price (MYR)"] * exchange_rate), 2)
product["Category"] = first_page.select(
"ol.breadcrumb > li.active")[0].text
product["Menu"] = first_page.select(
"li.primary-menu-item.selected > a > span")[0].text
product_list.append(product)
# Outer Loop iterates through list of generated URLs
for url in URL_list:
next_page = BS(session.get(url, headers=my_headers).content, "lxml")
# Inner loop iterates through elements on all other pages
for products in next_page.select("div.product-card"):
product = {}
product["Date"] = local_datetime.strftime("%Y/%m/%d")
product["Day"] = local_datetime.strftime("%a")
product["Territory"] = "Malaysia"
product["Menu Item"] = products.select("h5.product-title")[0].text
product["Price (MYR)"] = float((re.findall(
r"[-+]?(?:\d*\.\d+|\d+)", products.select("span.starting-price")[0].text)[0]))
product["Price (USD)"] = round(
(product["Price (MYR)"] * exchange_rate), 2)
product["Category"] = next_page.select(
"ol.breadcrumb > li.active")[0].text
product["Menu"] = next_page.select(
"li.primary-menu-item.selected > a > span")[0].text
product_list.append(product)
# ---------------------------------------------------- #
# Constructing the Dataframe and Exporting it to File #
# ---------------------------------------------------- #
product_list_df = pd.DataFrame(product_list)
product_list_df.index = pd.RangeIndex(
start=1, stop=(len(product_list_df.index) + 1), step=1)
print(product_list_df)
timestamp = str(local_datetime.strftime("[%Y-%m-%d %H:%M:%S]"))
product_list_df.to_csv(
f'./scraped-data/{str(timestamp + " mcd-bs4-my.csv")}', float_format="%.2f", encoding="utf-8")
# Output filename format: "[YYYY-MM-DD hh:mm:ss] mcd-bs4-my.csv"
except Exception:
print(
f'''
\n\n
---
One or more errors occurred:
{traceback.format_exc()}
---
\n\n
'''
)