Skip to content

Commit

Permalink
implement optional verbose output
Browse files Browse the repository at this point in the history
  • Loading branch information
gingeleski committed Mar 5, 2017
1 parent d04ba85 commit 6637f53
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 5 deletions.
15 changes: 13 additions & 2 deletions Scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,16 +40,27 @@ def parse_json(self, json_str):

return json.loads(json_str)

def scrape_all_urls(self):
def scrape_all_urls(self, do_verbose_output=False):
"""
Call the scrape method on every URL in this Scraper's league field, in
order, then close the browser.
Args:
do_verbose_output (bool): True/false do verbose output.
"""

if do_verbose_output is True:
output_str = "Start scraping " + self.league["league"] + " of "
output_str += self.league["area"] + "..."
print(output_str)

for url in self.league["urls"]:
self.scrape_url(url)
self.browser.close()

if do_verbose_output is True:
print("Done scraping this league.")

def scrape_url(self, url):
"""
Scrape the data for every match on a given URL and insert each into the
Expand Down Expand Up @@ -200,7 +211,7 @@ def get_scores(self, tag):
Returns:
(list of str) Extracted match scores.
"""

score_str = tag.find(class_="table-score").string
if self.is_invalid_game_from_score_string(score_str):
return [-1,-1]
Expand Down
6 changes: 3 additions & 3 deletions run.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
for possible_file in listdir(soccer_match_path):
if isfile(join(soccer_match_path, possible_file)):
soccer_match_json_file = join(soccer_match_path, possible_file)
with open(soccer_match_json_file, 'r') as open_json_file:
json_str = open_json_file.read().replace('\n', '')
with open(soccer_match_json_file, "r") as open_json_file:
json_str = open_json_file.read().replace("\n", "")
match_scraper = Scraper(json_str, initialize_db)
match_scraper.scrape_all_urls()
match_scraper.scrape_all_urls(True)
if initialize_db is True:
initialize_db = False

0 comments on commit 6637f53

Please sign in to comment.