scrape_poe_maps.py

#! python3
# -*- coding: utf-8 -*-
"""
# scrape_poe_maps.py - scrapes poe maps from http://pathofexile.gamepedia.com/Map
# (modified from scrape_poe_uniques.py)
"""

import requests, bs4, re, datetime, time, json
from bs4 import NavigableString
from multiprocessing.dummy import Pool as ThreadPool


base_url = 'http://pathofexile.gamepedia.com'
main_url = base_url + '/Map'

rx_search = re.compile(r'\+*\(([\d\.]+)\s[a-z]+\s([\d\.]+)[)]|(\+*[\d\.]+\%)|([\d\.]+-[\d\.]+)|(\([\d\.]+-[\d\.]+)\s\w+\s([\d\.]+-[\d\.]+\))|(-?\+?[\d\.]+)')
vendor_regex = re.compile('yields? one|produces? one', re.IGNORECASE)
maptype_regex = re.compile('Map type', re.IGNORECASE)


def write_file_headers():
    """
    info headers for MapList.txt

    :return: list
    """
    data = []
    d = datetime.datetime.now()
    now_time = d.strftime('%Y-%m-%d at %H:%M:%S')
    data.append('; Data from ' + main_url)
    data.append('; Comments can be made with ";", blank lines will be ignored.')
    data.append(';')
    data.append('; This file was auto-generated by scrape_poe_maps.py on {}'.format(now_time) + '\n')
    data.append('mapList := Object()')
    data.append('mapList["Unknown Map"] := "You have found a new map type, please report it!"\n')
    data.append('uniqueMapList := Object()')
    data.append('uniqueMapList["Unknown Map"] := "You have found a new unique map type, please report it!"\n')

    return data


def get_main_page(url):
    """
    Gets the main wiki page for Maps and parses out the links for each map
    and returns a list of the urls
    :param url: main page url
    :return: list of urls
    """
    map_urls = []
    print('Getting main page...')
    page = requests.get(url)
    page.raise_for_status()
    soup = bs4.BeautifulSoup(page.text, 'html.parser')
    map_table = soup.find_all('table', class_='wikitable')
    mapcount = 1
    for row in map_table[0].find_all('tr'):
        tds = row.find_all('td')
        if len(tds) == 0:    # exclude header row
            continue
        map_info = {}
        map_info['count'] = mapcount
        mapcount += 1
        map_info['name'] = tds[0].find('a').text.strip()
        map_info['level'] = tds[1].text.strip()
        map_info['tier'] = tds[2].text.strip()
        map_info['unique'] = tds[3].find('img')['alt'].strip()
        map_info['zone'] = tds[6].text.strip()
        map_info['url'] = base_url + tds[0].find('a')['href']
        map_info['base'] = None
        if map_info['unique'] == 'yes':
            count = 0
            for child in tds[0].find('span', class_='header').children:
                if child.string is not None:
                    count += 1
                if count == 2:
                    map_info['base'] = str(child).strip()
                    break
        else:
            map_info['base'] = map_info['name']

        map_urls.append(map_info)
    return map_urls


def get_category_url(data):  # extracts links from the category
    url_list = []
    page_url = 'http://pathofexile.gamepedia.com'
    for elem in data.find_all('a'):
        url_list.append(page_url + elem.attrs['href'])

    return url_list


def parse_map_data(map_info):
    """
    fetches the page for a map
    :param links:
    :return:
    """
    page = requests.get(map_info['url'])
    page.raise_for_status()
    soup = bs4.BeautifulSoup(page.text, 'html.parser')
    return build_data(soup, map_info)


def filter_unicode_string(str):
    return str.replace(u'\u2212', '-').replace(u'\u2013', '-').strip()


def append_modifier(results, mod, prepend):
    modifier = filter_unicode_string(mod)
    if len(modifier) > 0 and modifier.lower() != 'corrupted':
        results.append(prepend + modifier)


def parse_stats(stats, prepend):
    results = []
    for stat in stats:
        if isinstance(stat, NavigableString):
            append_modifier(results, stat, prepend)
        elif stat.name is not None and stat.name == 'span':
            append_modifier(results, stat.text, prepend)

    return results


def find_divcards(div):
    for h2 in div.find_all('h2'):
        for span in h2.find_all('span'):
            if span.text == 'Divination Cards':
                return h2
    return None


def find_vendor_recipe(div):
    for yields in div.find_all(text=vendor_regex):
        return yields.next_sibling.find('a').text
    return None


def find_setting(div):
    for maptype in div.find_all(text=maptype_regex):
        return maptype.parent.next_sibling.replace(':', '').strip()
    return None


def build_data(data, mapinfo):
    """
    parse map data from the page
    :param data: BS4 ResultSet
    :return: list
    """
    map_data = dict(mapinfo)
    print('Getting data for {}'.format(map_data['name'].encode('utf8')))

    div = data.find('div', id='mw-content-text')  # main div containing our data

    # find the h2 element containing "Divination Cards" and list items following it
    map_data['divcards'] = []
    div_h2 = find_divcards(div)
    if div_h2 is not None:
        divlist = div_h2.find_next_siblings('ul')
        if divlist is not None:
            divcards = divlist[0].find_all('li')
        for divcard in divcards:
            map_data['divcards'].append(divcard.find('a').text)

    # find the text string "yield|produce one" to determine the vendor recipe for the map
    map_data['vendor'] = find_vendor_recipe(div)

    # find the map setting (indoors/outdoors)
    map_data['setting'] = find_setting(div)

    #if map_data['unique'] == 'no' and map_data['tier'] != 'N/A' and int(map_data['tier']) != 15 and map_data['vendor'] is None:
    #    print('No vendor found for ' + map_data['name'])
    #if map_data['setting'] is None:
    #    print('No setting found for ' + map_data['name'])

    return map_data


def convert_data_to_AHK_readable_format(all_data):
    """
    This function takes the raw web page data, and converts it into lines that are readable by the
    Poe_item_info AHK script.
    :return:
    """

    # Read the descriptions of various maps
    with open('MapDescriptions.json', 'r') as f:
        map_descriptions = json.load(f)

    new_data = []
    vendor_map = {}
    unique_map = {}
    matchList = []
    for mymap in all_data:  # scan once to determine the lower tier map(s) that vendor up to each map
        if mymap['vendor'] is not None and mymap['unique'] == 'no':
            if not mymap['vendor'] in vendor_map:
                vendor_map[mymap['vendor']] = []
            vendor_map[mymap['vendor']].append(mymap['name'])
        if mymap['unique'] == 'no' and mymap['tier'] != 'N/A':
            matchList.append(mymap['name'])
        if mymap['unique'] == 'yes' and mymap['base'] is not None:
            unique_map[mymap['base']] = mymap['name']

    new_data.append('matchList := ["' + '","'.join(matchList) + '"]\n')

    for mymap in all_data:
        if mymap['base'] is None:
            print('ERROR: Could not determine base map type for map: ' + mymap['name'])
            continue
        structureName = 'mapList'
        if mymap['unique'] == 'yes':
            structureName = 'uniqueMapList'
            new_data.append(';' + mymap['name'])
        line = structureName + '["' + mymap['base'] + '"] := "`nThis map is based on the following area(s): '
        line += mymap['zone']
        if mymap['setting'] is not None:
            line += '`n`nMap type: ' + mymap['setting']

        # Here we insert the prewritten text descriptions for the maps that have them
        desc_key = 'maps'
        if mymap['unique'] == 'yes':
            desc_key = 'uniqueMaps'
        if mymap['base'] in map_descriptions[desc_key]:
            line += '`n`n' + map_descriptions[desc_key][mymap['base']]

        # Add on divination cards
        if len(mymap['divcards']) > 0:
            line += '`n`nDivination cards:'
            for dc in mymap['divcards']:
                line += '`n- ' + dc

        # Add on vendor recipes
        if mymap['unique'] == 'no':
            vendor_lines = []
            if mymap['base'] in vendor_map:
                for vend in vendor_map[mymap['base']]:
                    vendor_lines.append('`n- Produced by three ' + vend + 's')
            if mymap['vendor'] is not None:
                vendor_lines.append('`n- Three ' + mymap['base'] + 's produce one ' + mymap['vendor'])
            if len(vendor_lines) > 0:
                line += '`n`nVendor:'
                for vl in vendor_lines:
                    line += vl

        # Add on unique version if one exists
        if mymap['name'] in unique_map:
            line += '`n`nUnique version of map: ' + unique_map[mymap['name']]

        line += '"\n'
        new_data.append(line)

    return new_data


def write(new_data):
    file = open('MapList.txt', 'ab')  # opens file for writing
    for row in new_data:
        file.write(row.encode('utf8'))
        file.write(b'\n')
    file.close()


def main():
    map_list = get_main_page(main_url)
    open('MapList.txt', 'w').close()  # create file (or overwrite it if it exists)
    write(write_file_headers())
    pool = ThreadPool(4)
    data = pool.map(parse_map_data, map_list)
    data.sort(key=lambda m: m['count'])
    x = convert_data_to_AHK_readable_format(data)
    write(x)
    pool.close()
    pool.join()

startTime = datetime.datetime.now()
main()
print('Program execution time: ',(datetime.datetime.now() - startTime))