-
Notifications
You must be signed in to change notification settings - Fork 13
/
moira.py
311 lines (287 loc) · 11.6 KB
/
moira.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
"""MOIRA, the MOIRA Otto-matic Intelligent Reconniter of Assets, is an API for the Marketwatch Virtual Stock Exchange game.
Code is available on U{Github<http://github.com/brandonwu/moira>}.
"""
__docformat__ = "epytext en"
import requests
import logging
import re #Regex to clean up parsed out numbers
import json #Send/buy requests are JSON-encoded
from datetime import datetime
from dateutil.parser import parse
from dateutil import tz
from bs4 import BeautifulSoup
if __name__ == "__main__":
"""TODO: Put unit tests here or something.
@exclude:
"""
print("This is a Python module, to be imported into your own programs.\n"
'Use `import moira` in the interactive interpreter or your scripts.')
#Initialize logging
logger = logging.getLogger('default')
"""@exclude:"""
logger.propagate = False
logger.setLevel(logging.DEBUG)
#Initialize console output handler
ch = logging.StreamHandler()
fh = logging.FileHandler('moira.log', mode='w')
"""@exclude:"""
ch.setLevel(logging.INFO)
fh.setLevel(logging.DEBUG)
#Initialize log formatter
formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s')
"""@exclude:"""
ch.setFormatter(formatter)
fh.setFormatter(formatter)
logger.addHandler(ch)
logger.addHandler(fh)
#Timezone conversion
from_zone = tz.gettz('UTC')
to_zone = tz.gettz('America/New_York')
#Class it up so we'll be able to do portfolio.stock['EXCHANGETRADEDFUND-XASQ-IXJ'].price
#TODO: implement portfolio class
class Portfolio():
"""Stores portfolio data.
@param time: Last updated time (server time from HTTP headers).
@param cash: Amount of I{cash} (not purchasing power!) remaining.
@param leverage: Amount available to borrow.
@param net_worth: Sum of assets and liabilities.
@param purchasing_power: Amount (credit + cash) available to buy.
@param starting_cash: Cash amount provided at game start.
@param return_amt: Dollar amount of returns over L{starting_cash}.
"""
def __init__(self, time, cash, leverage, net_worth, purchasing_power,
starting_cash, return_amt, rank):
self.time = time
self.cash = cash
self.leverage = leverage
self.net_worth = net_worth
self.purchasing_power = purchasing_power
self.starting_cash = starting_cash
self.return_amt = return_amt
self.rank = rank
#TODO: implement purchase_price handling in Stock class
class Stock():
"""Stores portfolio data for a single stock.
@param id: Unique ID assigned by Marketwatch to each security.
@param ticker: The ticker symbol of the stock.
@param security_type: "ExchangeTradedFund" or "Stock"
@param current_price: Current price per share, I{rounded to the cent}.
@param shares: Number of shares held.
@param purchase_type: "Buy" or "Short"
@param returns: Total return on your investment.
@see See the warnings at L{get_current_holdings} about price rounding.
"""
def __init__(self, id, ticker, security_type, current_price, shares,
purchase_type, returns): #purchase_price
self.id = id
self.ticker = ticker
self.security_type = security_type
self.current_price = current_price
self.shares = shares
self.purchase_type = purchase_type
# self.purchase_price = purchase_price
self.returns = returns
class Trans():
"""Stores transaction data for a single transaction.
@param ticker: The ticker symbol of the security.
@param order_time: The time the order was issued.
@param trans_time: The time the order was executed.
@param trans_type: "Buy", "Short", "Sell", or "Cover"
@param trans_amt: Number of shares sold/purchased.
@param exec_price: Price of security at time of order.
"""
def __init__(self, ticker, order_time, trans_time, trans_type,
trans_amt, exec_price):
self.ticker = ticker
self.order_time = order_time
self.trans_time = trans_time
self.trans_type = trans_type
self.trans_amt = trans_amt
self.exec_price = exec_price
def get_token(username, password, returnsession=False, s=requests.Session()):
"""Issues a login request. The token returned by this function
is required for all methods in this module.
@param username: The marketwatch.com username (email).
@param password: The plaintext marketwatch.com password.
@return: Requests cookiejar containing authentication token.
@note: It's unknown what the expiry time for this token is - it is
set to expire at end of session. It may be apt to request
a new token daily, while the market is closed.
"""
s.get('http://www.marketwatch.com/')
datata = s.get('https://id.marketwatch.com/auth/submitlogin.json',params={'username': username, 'password': password})
s.get(datata.json()['url'])
#TODO: Turn this into something that checks the cookiejar for .ASPXAUTH instead; this takes WAY too long.
if s.get('http://www.marketwatch.com/user/login/status').url == \
"http://www.marketwatch.com/my":
logger.info("Login success.")
else:
logger.warn("Auth failure.")
if returnsession:
return s.cookies, s#methods that take tokens can take sessions
#but the session is optional while the token is not
else:
return s.cookies
def get_current_holdings(token, game, s=requests.Session()):
"""Fetches and parses current holdings.
@param token: Cookiejar returned by a call to L{get_token}.
@param game: The X{name} of the game (marketwatch.com/game/I{XXXXXXX}).
@return: L{Stock} data.
@rtype: Dict of L{Stock} objects, keyed by X{id}
@warning: The stock price returned by a call to L{get_current_holdings}
is rounded to the nearest cent! This results in inaccuracies if you
calculate things based on this number --- don't. Use L{stock_search}
instead. Interestingly, Marketwatch itself never reports the full-
precision stock price anywhere except in HTML attributes.
"""
url = ("http://www.marketwatch.com/game/%s/portfolio/holdings"
"?view=list&partial=True") % game
p = s
r = p.get(url, cookies=token)
time = parse(r.headers['date'])
time = time.replace(tzinfo=from_zone).astimezone(to_zone)
response = r.text
soup = BeautifulSoup(response)
#the trs in portf. page have stock data atributes in the tag
trs = soup.find_all('tr')
trs.pop(0) #remove table header row
#get tds for current ROI per stock
tds = soup.find_all('td', {'class': 'marketgain'})
#TODO: get purchase_price by parsing orders page
#assemble stock objects
#stock objects are stored in a dict keyed by the stock id
stock_dict = {}
for x,y in zip(trs, tds):
o = Stock(x['data-symbol'], x['data-ticker'],
x['data-insttype'], float(x['data-price']),
int(float(x['data-shares'])), x['data-type'],
float(re.sub("\r\n\t*", "", y.contents[0]). \
replace(',','')) #TODO: incl purchase price
)
stock_dict[x['data-symbol']] = o
#p = Portfolio()
#access the data in the stock dict returned like so:
#stock_dict['EXCHANGETRADEDFUND-XASQ-IXJ'].current_price
return stock_dict
def get_transaction_history(token, game, s=requests.Session()):
"""DOES NOT FUNCTION YET: Downloads and parses the list of past transactions.
@param token: Cookiejar returned by L{get_token}.
@param game: The X{name} of the game (marketwatch.com/game/I{XXXXXXX}).
@return: A dict of all past transactions.
@rtype: Dict of L{Trans} objects, keyed on an index (1, 2...).
"""
orders_url = ("http://www.marketwatch.com/game/msj-2013/portfolio/"
"transactionhistory?sort=TransactionDate&descending="
"True&partial=true&index=%s")
soup = BeautifulSoup(s.get(orders_url, cookies=token).text)
total = int(soup.find('a',{'class':'fakebutton'})['href'].
split('&')[1].split('=')[1])
if total >= 10:
whole = int(str(total)[0:-1])*10
else:
whole = 0
tail = total - whole
for i in range(0, whole, 10):
r = s.get(orders_url % i, token)
ordersoup = BeautifulSoup(r)
def stock_search(token, game, ticker, s=requests.Session()):
"""Queries Marketwatch for stock price and ID of a given ticker symbol.
@note: You must have joined a game in order to use this function.
@param token: Cookiejar returned by L{get_token}.
@param game: Game name (marketwatch.com/game/I{XXXXXXX}).
@param ticker: Ticker symbol of stock to query.
@return: Current stock price, stock id, and server time.
@rtype: Dict {'price':float,
'id':str,
'time':I{datetime} object in EST}.
"""
search_url = 'http://www.marketwatch.com/game/%s/trade?week=1' % game
postdata = {'search': ticker, 'view': 'grid', 'partial': 'true'}
r = s.post(search_url, data=postdata, cookies=token)
if not r.status_code == 200:
logger.error('Server returned HTTP %s; probably rate-limiting.' \
% r.status_code)
return 1
soup = BeautifulSoup(r.text)
time = parse(r.headers['date'])
time = time.replace(tzinfo=from_zone).astimezone(to_zone)
try:
price = float(soup.find('div',{'class': 'chip'})['data-price'])
symbol = soup.find('div',{'class': 'chip'})['data-symbol']
dict = {'price': price, 'id': symbol, 'time': time}
return dict
except TypeError:
logger.error('Invalid game or Marketwatch rate-limiting.')
logger.debug(r.headers)
logger.debug(r.text)
pass
except KeyError:
logger.error('Invalid stock ticker symbol provided, or '
'Marketwatch rate-limiting.')
logger.debug(r.headers)
logger.debug(r.text)
pass
except Exception,e:
logger.error(repr(e))
logger.debug(r.headers)
logger.debug(r.text)
pass
def get_portfolio_data(token, game, s=requests.Session()):
"""Grabs portfolio data.
@param token: Cookiejar returned by L{get_token}.
@param game: Game name (marketwatch.com/game/I{XXXXXXX})
@return: Portfolio data dictionary
@rtype: Dict with net_worth, overall_return_amount, overall_return_percent,
daily_return_percent, purchasing_power, cash_left, cash_borrowed,
short_reserve, rank, and time (last updated).
@note: I probably won't be making this return a L{Portfolio} object; it seems
slightly redundant.
"""
portfolio_url = "http://www.marketwatch.com/game/%s/portfolio" % game
d = s.get(portfolio_url, cookies=token)
r = BeautifulSoup(d.text)
time = datetime.strptime(d.headers['date'],'%a, %d %b %Y %H:%M:%S %Z')
time = time.replace(tzinfo=from_zone).astimezone(to_zone)
rank = r.find('p', {'class': 'rank'}).contents[0]
if rank != '--':
rank = int(rank)
data = r.find_all('span', {'class': 'data'})
data = [ float(re.sub('[^-\d\.]', '', x.contents[0])) for x in data ]
data_keys = ['net_worth', 'overall_return_amount', 'overall_return_percent',
'daily_return_percent', 'purchasing_power', 'cash_left',
'cash_borrowed', 'short_reserve']
data = dict(zip(data_keys, data))
data.update({'rank': rank, 'time': time})
return data
def order(token, game, type, id, amt, s=requests.Session()):
"""Initiates a buy, sell, short, or cover order.
@warning: If you have insufficient funds, the server will still
respond that the order succeeded! Check the order and
transaction list to make sure the order actually went
through.
@param token: Cookiejar returned by L{get_token}.
@param game: Game name (marketwatch.com/game/I{XXXXXXX})
@param id: Security ID (not the ticker symbol).
Obtain from L{stock_search}
@param amt: Order amount.
@param type: Type of order - 'Sell', 'Buy', 'Short', or 'Cover'.
@return: Returns integer - 0 if success, nonzero if failure.
@rtype: integer
"""
order_url = 'http://www.marketwatch.com/game/%s/trade/' \
'submitorder?week=1' % game
postdata = '['+json.dumps({'Fuid': id, 'Shares': str(amt), \
'Type': type})+']'
headers = {'X-Requested-With': 'XMLHttpRequest',
'Content-Type': 'application/json',
'charset': 'UTF-8'}
resp = json.loads(s.post(order_url, data=postdata, cookies=token,
headers=headers).text)
if resp['succeeded'] == True:
logger.info('Order may have succeeded. Server said: %s' \
% resp['message'])
return True, resp['message']
else:
logger.error('Order failed. Server said: %s' \
% resp['message'])
return False, resp['message']