From 30ec40589090c56664d20feeb0f4e1ff1cbf7a61 Mon Sep 17 00:00:00 2001 From: Vehicle Researcher Date: Mon, 10 Dec 2018 14:06:10 -0800 Subject: [PATCH] Squashed 'pyextra/' changes from fb152de..8cc1594 8cc1594 update overpy 2c18997 switch to forked version of overpy git-subtree-dir: pyextra git-subtree-split: 8cc1594238fcae6acb8bcd825ac73bbca586b1be --- overpy-0.4-py2.7.egg-info/PKG-INFO | 123 ---------- overpy-0.4-py2.7.egg-info/SOURCES.txt | 61 ----- .../dependency_links.txt | 1 - overpy-0.4-py2.7.egg-info/installed-files.txt | 56 ----- overpy-0.4-py2.7.egg-info/not-zip-safe | 1 - overpy-0.4-py2.7.egg-info/top_level.txt | 2 - overpy/__init__.py | 221 ++++++++++++------ overpy/exception.py | 58 +++++ 8 files changed, 210 insertions(+), 313 deletions(-) delete mode 100644 overpy-0.4-py2.7.egg-info/PKG-INFO delete mode 100644 overpy-0.4-py2.7.egg-info/SOURCES.txt delete mode 100644 overpy-0.4-py2.7.egg-info/dependency_links.txt delete mode 100644 overpy-0.4-py2.7.egg-info/installed-files.txt delete mode 100644 overpy-0.4-py2.7.egg-info/not-zip-safe delete mode 100644 overpy-0.4-py2.7.egg-info/top_level.txt diff --git a/overpy-0.4-py2.7.egg-info/PKG-INFO b/overpy-0.4-py2.7.egg-info/PKG-INFO deleted file mode 100644 index 96d494a147e90f..00000000000000 --- a/overpy-0.4-py2.7.egg-info/PKG-INFO +++ /dev/null @@ -1,123 +0,0 @@ -Metadata-Version: 1.1 -Name: overpy -Version: 0.4 -Summary: Python Wrapper to access the OpenStreepMap Overpass API -Home-page: https://github.com/DinoTools/python-overpy -Author: PhiBo (DinoTools) -Author-email: UNKNOWN -License: MIT -Description: Python Overpass Wrapper - ======================= - - A Python Wrapper to access the Overpass API. - - Have a look at the `documentation`_ to find additional information. - - .. image:: https://pypip.in/version/overpy/badge.svg - :target: https://pypi.python.org/pypi/overpy/ - :alt: Latest Version - - .. image:: https://pypip.in/license/overpy/badge.svg - :target: https://pypi.python.org/pypi/overpy/ - :alt: License - - .. image:: https://travis-ci.org/DinoTools/python-overpy.svg?branch=master - :target: https://travis-ci.org/DinoTools/python-overpy - - .. image:: https://coveralls.io/repos/DinoTools/python-overpy/badge.png?branch=master - :target: https://coveralls.io/r/DinoTools/python-overpy?branch=master - - Features - -------- - - * Query Overpass API - * Parse JSON and XML response data - * Additional helper functions - - Install - ------- - - **Requirements:** - - Supported Python versions: - - * Python 2.7 - * Python >= 3.2 - * PyPy and PyPy3 - - **Install:** - - .. code-block:: console - - $ pip install overpy - - Examples - -------- - - Additional examples can be found in the `documentation`_ and in the *examples* directory. - - .. code-block:: python - - import overpy - - api = overpy.Overpass() - - # fetch all ways and nodes - result = api.query(""" - way(50.746,7.154,50.748,7.157) ["highway"]; - (._;>;); - out body; - """) - - for way in result.ways: - print("Name: %s" % way.tags.get("name", "n/a")) - print(" Highway: %s" % way.tags.get("highway", "n/a")) - print(" Nodes:") - for node in way.nodes: - print(" Lat: %f, Lon: %f" % (node.lat, node.lon)) - - - Helper - ~~~~~~ - - Helper methods are available to provide easy access to often used requests. - - .. code-block:: python - - import overpy.helper - - # 3600062594 is the OSM id of Chemnitz and is the bounding box for the request - street = overpy.helper.get_street( - "Straße der Nationen", - "3600062594" - ) - - # this finds an intersection between Straße der Nationen and Carolastraße in Chemnitz - intersection = overpy.helper.get_intersection( - "Straße der Nationen", - "Carolastraße", - "3600062594" - ) - - - License - ------- - - Published under the MIT (see LICENSE for more information) - - .. _`documentation`: http://python-overpy.readthedocs.org/ - -Keywords: OverPy Overpass OSM OpenStreetMap -Platform: UNKNOWN -Classifier: Development Status :: 4 - Beta -Classifier: License :: OSI Approved :: MIT License -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.2 -Classifier: Programming Language :: Python :: 3.3 -Classifier: Programming Language :: Python :: 3.4 -Classifier: Programming Language :: Python :: 3.5 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy diff --git a/overpy-0.4-py2.7.egg-info/SOURCES.txt b/overpy-0.4-py2.7.egg-info/SOURCES.txt deleted file mode 100644 index 66bf17f7c4d393..00000000000000 --- a/overpy-0.4-py2.7.egg-info/SOURCES.txt +++ /dev/null @@ -1,61 +0,0 @@ -CHANGELOG.rst -LICENSE -MANIFEST.in -README.rst -setup.cfg -setup.py -docs/make.bat -docs/source/api.rst -docs/source/changelog.rst -docs/source/conf.py -docs/source/contributing.rst -docs/source/example.rst -docs/source/index.rst -docs/source/introduction.rst -examples/get_areas.py -examples/get_nodes.py -examples/get_ways.py -overpy/__about__.py -overpy/__init__.py -overpy/exception.py -overpy/helper.py -overpy.egg-info/PKG-INFO -overpy.egg-info/SOURCES.txt -overpy.egg-info/dependency_links.txt -overpy.egg-info/not-zip-safe -overpy.egg-info/top_level.txt -tests/__init__.py -tests/base_class.py -tests/test_exception.py -tests/test_json.py -tests/test_request.py -tests/test_result.py -tests/test_result_way.py -tests/test_xml.py -tests/json/area-01.json -tests/json/node-01.json -tests/json/relation-01.json -tests/json/relation-02.json -tests/json/relation-03.json -tests/json/relation-04.json -tests/json/result-expand-01.json -tests/json/result-expand-02.json -tests/json/result-way-01.json -tests/json/result-way-02.json -tests/json/result-way-03.json -tests/json/way-01.json -tests/json/way-02.json -tests/json/way-03.json -tests/json/way-04.json -tests/response/bad-request-encoding.html -tests/response/bad-request.html -tests/xml/area-01.xml -tests/xml/node-01.xml -tests/xml/relation-01.xml -tests/xml/relation-02.xml -tests/xml/relation-03.xml -tests/xml/relation-04.xml -tests/xml/way-01.xml -tests/xml/way-02.xml -tests/xml/way-03.xml -tests/xml/way-04.xml \ No newline at end of file diff --git a/overpy-0.4-py2.7.egg-info/dependency_links.txt b/overpy-0.4-py2.7.egg-info/dependency_links.txt deleted file mode 100644 index 8b137891791fe9..00000000000000 --- a/overpy-0.4-py2.7.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/overpy-0.4-py2.7.egg-info/installed-files.txt b/overpy-0.4-py2.7.egg-info/installed-files.txt deleted file mode 100644 index 0cd05ada1f7c8d..00000000000000 --- a/overpy-0.4-py2.7.egg-info/installed-files.txt +++ /dev/null @@ -1,56 +0,0 @@ -../overpy/__about__.py -../overpy/__about__.pyc -../overpy/__init__.py -../overpy/__init__.pyc -../overpy/exception.py -../overpy/exception.pyc -../overpy/helper.py -../overpy/helper.pyc -../tests/__init__.py -../tests/__init__.pyc -../tests/base_class.py -../tests/base_class.pyc -../tests/json/area-01.json -../tests/json/node-01.json -../tests/json/relation-01.json -../tests/json/relation-02.json -../tests/json/relation-03.json -../tests/json/relation-04.json -../tests/json/result-expand-01.json -../tests/json/result-expand-02.json -../tests/json/result-way-01.json -../tests/json/result-way-02.json -../tests/json/result-way-03.json -../tests/json/way-01.json -../tests/json/way-02.json -../tests/json/way-03.json -../tests/json/way-04.json -../tests/response/bad-request-encoding.html -../tests/response/bad-request.html -../tests/test_exception.py -../tests/test_exception.pyc -../tests/test_json.py -../tests/test_json.pyc -../tests/test_request.py -../tests/test_request.pyc -../tests/test_result.py -../tests/test_result.pyc -../tests/test_result_way.py -../tests/test_result_way.pyc -../tests/test_xml.py -../tests/test_xml.pyc -../tests/xml/area-01.xml -../tests/xml/node-01.xml -../tests/xml/relation-01.xml -../tests/xml/relation-02.xml -../tests/xml/relation-03.xml -../tests/xml/relation-04.xml -../tests/xml/way-01.xml -../tests/xml/way-02.xml -../tests/xml/way-03.xml -../tests/xml/way-04.xml -PKG-INFO -SOURCES.txt -dependency_links.txt -not-zip-safe -top_level.txt diff --git a/overpy-0.4-py2.7.egg-info/not-zip-safe b/overpy-0.4-py2.7.egg-info/not-zip-safe deleted file mode 100644 index 8b137891791fe9..00000000000000 --- a/overpy-0.4-py2.7.egg-info/not-zip-safe +++ /dev/null @@ -1 +0,0 @@ - diff --git a/overpy-0.4-py2.7.egg-info/top_level.txt b/overpy-0.4-py2.7.egg-info/top_level.txt deleted file mode 100644 index 4611d9bb608014..00000000000000 --- a/overpy-0.4-py2.7.egg-info/top_level.txt +++ /dev/null @@ -1,2 +0,0 @@ -overpy -tests diff --git a/overpy/__init__.py b/overpy/__init__.py index 8e038bda96bd02..2836080ab7d017 100644 --- a/overpy/__init__.py +++ b/overpy/__init__.py @@ -5,6 +5,8 @@ import json import re import sys +import time +import requests from overpy import exception from overpy.__about__ import ( @@ -18,12 +20,15 @@ XML_PARSER_DOM = 1 XML_PARSER_SAX = 2 -if PY2: - from urllib2 import urlopen - from urllib2 import HTTPError -elif PY3: - from urllib.request import urlopen - from urllib.error import HTTPError +# Try to convert some common attributes +# http://wiki.openstreetmap.org/wiki/Elements#Common_attributes +GLOBAL_ATTRIBUTE_MODIFIERS = { + "changeset": int, + "timestamp": lambda ts: datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ"), + "uid": int, + "version": int, + "visible": lambda v: v.lower() == "true" +} def is_valid_type(element, cls): @@ -41,11 +46,16 @@ def is_valid_type(element, cls): class Overpass(object): """ Class to access the Overpass API + + :cvar default_max_retry_count: Global max number of retries (Default: 0) + :cvar default_retry_timeout: Global time to wait between tries (Default: 1.0s) """ + default_max_retry_count = 0 default_read_chunk_size = 4096 + default_retry_timeout = 1.0 default_url = "http://overpass-api.de/api/interpreter" - def __init__(self, read_chunk_size=None, url=None, xml_parser=XML_PARSER_SAX): + def __init__(self, read_chunk_size=None, url=None, xml_parser=XML_PARSER_SAX, max_retry_count=None, retry_timeout=None, timeout=5.0, headers=None): """ :param read_chunk_size: Max size of each chunk read from the server response :type read_chunk_size: Integer @@ -53,6 +63,14 @@ def __init__(self, read_chunk_size=None, url=None, xml_parser=XML_PARSER_SAX): :type url: str :param xml_parser: The xml parser to use :type xml_parser: Integer + :param max_retry_count: Max number of retries (Default: default_max_retry_count) + :type max_retry_count: Integer + :param retry_timeout: Time to wait between tries (Default: default_retry_timeout) + :type retry_timeout: float + :param timeout: HTTP request timeout + :type timeout: float + :param headers: HTTP request headers + :type headers: dict """ self.url = self.default_url if url is not None: @@ -63,7 +81,34 @@ def __init__(self, read_chunk_size=None, url=None, xml_parser=XML_PARSER_SAX): if read_chunk_size is None: read_chunk_size = self.default_read_chunk_size self.read_chunk_size = read_chunk_size + + if max_retry_count is None: + max_retry_count = self.default_max_retry_count + self.max_retry_count = max_retry_count + + if retry_timeout is None: + retry_timeout = self.default_retry_timeout + self.retry_timeout = retry_timeout + self.xml_parser = xml_parser + self.timeout = timeout + self.headers = headers + + def _handle_remark_msg(self, msg): + """ + Try to parse the message provided with the remark tag or element. + + :param str msg: The message + :raises overpy.exception.OverpassRuntimeError: If message starts with 'runtime error:' + :raises overpy.exception.OverpassRuntimeRemark: If message starts with 'runtime remark:' + :raises overpy.exception.OverpassUnknownError: If we are unable to identify the error + """ + msg = msg.strip() + if msg.startswith("runtime error:"): + raise exception.OverpassRuntimeError(msg=msg) + elif msg.startswith("runtime remark:"): + raise exception.OverpassRuntimeRemark(msg=msg) + raise exception.OverpassUnknownError(msg=msg) def query(self, query): """ @@ -76,56 +121,79 @@ def query(self, query): if not isinstance(query, bytes): query = query.encode("utf-8") - try: - f = urlopen(self.url, query) - except HTTPError as e: - f = e - - response = f.read(self.read_chunk_size) - while True: - data = f.read(self.read_chunk_size) - if len(data) == 0: - break - response = response + data - f.close() - - if f.code == 200: - if PY2: - http_info = f.info() - content_type = http_info.getheader("content-type") - else: - content_type = f.getheader("Content-Type") - - if content_type == "application/json": - return self.parse_json(response) + retry_num = 0 + retry_exceptions = [] + do_retry = True if self.max_retry_count > 0 else False + while retry_num <= self.max_retry_count: + if retry_num > 0: + time.sleep(self.retry_timeout) + retry_num += 1 - if content_type == "application/osm3s+xml": - return self.parse_xml(response) + try: + if self.headers is not None: + r = requests.post(self.url, query, timeout=self.timeout, headers=self.headers) + else: + r = requests.post(self.url, query, timeout=self.timeout) + response = r.content + except (requests.exceptions.BaseHTTPError, requests.exceptions.RequestException) as e: + if not do_retry: + raise e + retry_exceptions.append(e) + continue - raise exception.OverpassUnknownContentType(content_type) + if r.status_code == 200: + content_type = r.headers["Content-Type"] - if f.code == 400: - msgs = [] - for msg in self._regex_extract_error_msg.finditer(response): - tmp = self._regex_remove_tag.sub(b"", msg.group("msg")) - try: - tmp = tmp.decode("utf-8") - except UnicodeDecodeError: - tmp = repr(tmp) - msgs.append(tmp) + if content_type == "application/json": + return self.parse_json(response) - raise exception.OverpassBadRequest( - query, - msgs=msgs - ) + if content_type == "application/osm3s+xml": + return self.parse_xml(response) - if f.code == 429: - raise exception.OverpassTooManyRequests + e = exception.OverpassUnknownContentType(content_type) + if not do_retry: + raise e + retry_exceptions.append(e) + continue + elif r.status_code == 400: + msgs = [] + for msg in self._regex_extract_error_msg.finditer(response): + tmp = self._regex_remove_tag.sub(b"", msg.group("msg")) + try: + tmp = tmp.decode("utf-8") + except UnicodeDecodeError: + tmp = repr(tmp) + msgs.append(tmp) + + e = exception.OverpassBadRequest( + query, + msgs=msgs + ) + if not do_retry: + raise e + retry_exceptions.append(e) + continue + elif r.status_code == 429: + e = exception.OverpassTooManyRequests + if not do_retry: + raise e + retry_exceptions.append(e) + continue + elif r.status_code == 504: + e = exception.OverpassGatewayTimeout + if not do_retry: + raise e + retry_exceptions.append(e) + continue - if f.code == 504: - raise exception.OverpassGatewayTimeout + # No valid response code + e = exception.OverpassUnknownHTTPStatusCode(r.status_code) + if not do_retry: + raise e + retry_exceptions.append(e) + continue - raise exception.OverpassUnknownHTTPStatusCode(f.code) + raise exception.MaxRetriesReached(retry_count=retry_num, exceptions=retry_exceptions) def parse_json(self, data, encoding="utf-8"): """ @@ -139,8 +207,11 @@ def parse_json(self, data, encoding="utf-8"): :rtype: overpy.Result """ if isinstance(data, bytes): - data = data.decode(encoding) + data = data.decode(encoding) + data = json.loads(data, parse_float=Decimal) + if "remark" in data: + self._handle_remark_msg(msg=data.get("remark")) return Result.from_json(data, api=self) def parse_xml(self, data, encoding="utf-8", parser=None): @@ -155,13 +226,16 @@ def parse_xml(self, data, encoding="utf-8", parser=None): """ if parser is None: parser = self.xml_parser - if isinstance(data, bytes): data = data.decode(encoding) if PY2 and not isinstance(data, str): # Python 2.x: Convert unicode strings data = data.encode(encoding) + m = re.compile("(?P[^<>]*)").search(data) + if m: + self._handle_remark_msg(m.group("msg")) + return Result.from_xml(data, api=self, parser=parser) @@ -279,23 +353,39 @@ def from_json(cls, data, api=None): return result @classmethod - def from_xml(cls, data, api=None, parser=XML_PARSER_SAX): + def from_xml(cls, data, api=None, parser=None): """ - Create a new instance and load data from xml object. + Create a new instance and load data from xml data or object. + + .. note:: + If parser is set to None, the functions tries to find the best parse. + By default the SAX parser is chosen if a string is provided as data. + The parser is set to DOM if an xml.etree.ElementTree.Element is provided as data value. :param data: Root element - :type data: xml.etree.ElementTree.Element - :param api: + :type data: str | xml.etree.ElementTree.Element + :param api: The instance to query additional information if required. :type api: Overpass - :param parser: Specify the parser to use(DOM or SAX) - :type parser: Integer + :param parser: Specify the parser to use(DOM or SAX)(Default: None = autodetect, defaults to SAX) + :type parser: Integer | None :return: New instance of Result object :rtype: Result """ + if parser is None: + if isinstance(data, str): + parser = XML_PARSER_SAX + else: + parser = XML_PARSER_DOM + result = cls(api=api) if parser == XML_PARSER_DOM: import xml.etree.ElementTree as ET - root = ET.fromstring(data) + if isinstance(data, str): + root = ET.fromstring(data) + elif isinstance(data, ET.Element): + root = data + else: + raise exception.OverPyException("Unable to detect data type.") for elem_cls in [Node, Way, Relation, Area]: for child in root: @@ -522,17 +612,10 @@ def __init__(self, attributes=None, result=None, tags=None): """ self._result = result - # Try to convert some common attributes - # http://wiki.openstreetmap.org/wiki/Elements#Common_attributes - self._attribute_modifiers = { - "changeset": int, - "timestamp": lambda ts: datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ"), - "uid": int, - "version": int, - "visible": lambda v: v.lower() == "true" - } self.attributes = attributes - for n, m in self._attribute_modifiers.items(): + # ToDo: Add option to modify attribute modifiers + attribute_modifiers = dict(GLOBAL_ATTRIBUTE_MODIFIERS.items()) + for n, m in attribute_modifiers.items(): if n in self.attributes: self.attributes[n] = m(self.attributes[n]) self.id = None diff --git a/overpy/exception.py b/overpy/exception.py index 7179d246a59015..3d8416a1257ef1 100644 --- a/overpy/exception.py +++ b/overpy/exception.py @@ -37,6 +37,18 @@ def __str__(self): ) +class MaxRetriesReached(OverPyException): + """ + Raised if max retries reached and the Overpass server didn't respond with a result. + """ + def __init__(self, retry_count, exceptions): + self.exceptions = exceptions + self.retry_count = retry_count + + def __str__(self): + return "Unable get any result from the Overpass API server after %d retries." % self.retry_count + + class OverpassBadRequest(OverPyException): """ Raised if the Overpass API service returns a syntax error. @@ -62,6 +74,29 @@ def __str__(self): return "\n".join(tmp_msgs) +class OverpassError(OverPyException): + """ + Base exception to report errors if the response returns a remark tag or element. + + .. note:: + If you are not sure which of the subexceptions you should use, use this one and try to parse the message. + + For more information have a look at https://github.com/DinoTools/python-overpy/issues/62 + + :param str msg: The message from the remark tag or element + """ + def __init__(self, msg=None): + #: The message from the remark tag or element + self.msg = msg + + def __str__(self): + if self.msg is None: + return "No error message provided" + if not isinstance(self.msg, str): + return str(self.msg) + return self.msg + + class OverpassGatewayTimeout(OverPyException): """ Raised if load of the Overpass API service is too high and it can't handle the request. @@ -70,6 +105,22 @@ def __init__(self): OverPyException.__init__(self, "Server load too high") +class OverpassRuntimeError(OverpassError): + """ + Raised if the server returns a remark-tag(xml) or remark element(json) with a message starting with + 'runtime error:'. + """ + pass + + +class OverpassRuntimeRemark(OverpassError): + """ + Raised if the server returns a remark-tag(xml) or remark element(json) with a message starting with + 'runtime remark:'. + """ + pass + + class OverpassTooManyRequests(OverPyException): """ Raised if the Overpass API service returns a 429 status code. @@ -94,6 +145,13 @@ def __str__(self): return "Unknown content type: %s" % self.content_type +class OverpassUnknownError(OverpassError): + """ + Raised if the server returns a remark-tag(xml) or remark element(json) and we are unable to find any reason. + """ + pass + + class OverpassUnknownHTTPStatusCode(OverPyException): """ Raised if the returned HTTP status code isn't handled by OverPy.