-
Notifications
You must be signed in to change notification settings - Fork 0
/
scrape_tests.py
86 lines (62 loc) · 2.56 KB
/
scrape_tests.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
from bs4 import BeautifulSoup
import requests
from pprint import pprint
import yfinance as yf
def get_sp_tickers():
# Get sp500 ticker and sector
url = 'https://en.wikipedia.org/wiki/List_of_S%26P_500_companies'
response = requests.get(url)
soup = BeautifulSoup(response.content, 'html.parser')
table = soup.find('table', {'class': 'wikitable sortable'})
rows = table.find_all('tr')[1:] # skip the header row
sp500 = []
for row in rows:
cells = row.find_all('td')
ticker = cells[0].text.strip()
company = cells[1].text.strip()
sector = cells[3].text.strip()
sp500.append({'ticker': ticker, 'company': company, 'sector': sector})
return sp500
def get_sp500_stocks(sp500):
sp500_stocks = []
for stock in sp500:
try:
print(stock['ticker'])
#price = get_stock_price(stock['ticker'])
price = get_stock_price2(stock['ticker'])
print(price)
sp500_stocks.append((stock['ticker'], stock['sector'], price))
print(sp500_stocks)
except:
print((f"There was an issue with {stock['ticker']}."))
return sp500_stocks
def get_headers():
return {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36"}
def get_stock_price(ticker):
try:
url = f'https://finance.yahoo.com/quote/{ticker}'
response = requests.get(url, headers=get_headers())
soup = BeautifulSoup(response.content, 'html.parser')
data = soup.find('fin-streamer', {'data-symbol': ticker})
price = float(data['value'])
return price
except:
print(f'Price not available for {ticker}')
price = 0.0
return price
def get_stock_price2(ticker):
stock = yf.Ticker(ticker)
info = stock.info
try:
if 'currentPrice' in info:
price = info['currentPrice'] # use currentPrice or regularMarketPrice
return price
else:
print(f"Current price not available for {ticker}")
return None
except:
print(f'Current price not available for {ticker}')
return price
if __name__ == "__main__":
sp500 = get_sp_tickers()
get_sp500_stocks(sp500)