Allow xml write/read of additional attributes

genet/__init__.py

@@ -1,7 +1,7 @@
 from genet.core import Network  # noqa: F401
 from genet.schedule_elements import Schedule, Service, Route, Stop  # noqa: F401
 from genet.use.road_pricing import Toll  # noqa: F401
-from genet.inputs_handler.read import *  # noqa: F401,F403
+from genet.input.read import *  # noqa: F401,F403
 from genet.auxiliary_files import AuxiliaryFile  # noqa: F401
 from genet.max_stable_set import MaxStableSet  # noqa: F401
 from genet.exceptions import ScheduleElementGraphSchemaError, RouteInitialisationError, \

genet/configs/OSM/default_config.yml

@@ -111,7 +111,7 @@ MODES:
 
   DEFAULT_OSM_TAG_VALUE:
   # GeNet will first look at OSM tags to infer the matsim values the link should have. For the types and values head over to
-  # outputs_handler/matsim_xml_values.py.
+  # output/matsim_xml_values.py.
     car: secondary
     bus: secondary
     rail: railway

genet/configs/OSM/slim_config.yml

@@ -67,7 +67,7 @@ MODES:
 
   DEFAULT_OSM_TAG_VALUE:
   # GeNet will first look at OSM tags to infer the matsim values the link should have. For the types and values head over to
-  # outputs_handler/matsim_xml_values.py.
+  # output/matsim_xml_values.py.
     car: secondary
     bus: secondary
     rail: railway

genet/core.py

@@ -12,9 +12,9 @@
 import genet.modify.change_log as change_log
 import genet.modify.graph as modify_graph
 import genet.modify.schedule as modify_schedule
-import genet.outputs_handler.geojson as geojson
-import genet.outputs_handler.matsim_xml_writer as matsim_xml_writer
-import genet.outputs_handler.sanitiser as sanitiser
+import genet.output.geojson as geojson
+import genet.output.matsim_xml_writer as matsim_xml_writer
+import genet.output.sanitiser as sanitiser
 import genet.schedule_elements as schedule_elements
 import genet.utils.dict_support as dict_support
 import genet.utils.elevation as elevation
@@ -37,16 +37,19 @@
 
 
 class Network:
-    def __init__(self, epsg):
+    def __init__(self, epsg, **kwargs):
         self.epsg = epsg
         self.transformer = Transformer.from_crs(epsg, 'epsg:4326', always_xy=True)
-        self.graph = nx.MultiDiGraph(name='Network graph', crs=self.epsg, simplified=False)
+        self.graph = nx.MultiDiGraph(name='Network graph', crs=epsg)
+        self.attributes = {'crs': {'name': 'crs', 'class': 'java.lang.String', 'text': epsg}}
         self.schedule = schedule_elements.Schedule(epsg)
         self.change_log = change_log.ChangeLog()
         self.auxiliary_files = {'node': {}, 'link': {}}
         # link_id_mapping maps between (usually string literal) index per edge to the from and to nodes that are
         # connected by the edge
         self.link_id_mapping = {}
+        if kwargs:
+            self.add_additional_attributes(kwargs)
 
     def __repr__(self):
         return f"<{self.__class__.__name__} instance at {id(self)}: with \ngraph: {nx.info(self.graph)} and " \
@@ -55,9 +58,22 @@ def __repr__(self):
     def __str__(self):
         return self.info()
 
+    def add_additional_attributes(self, attribs: dict):
+        """
+        adds attributes defined by keys of the attribs dictionary with values of the corresponding values
+        :param attribs: the additional attributes {attribute_name: attribute_value}
+        :return:
+        """
+        for k, v in attribs.items():
+            if k not in self.__dict__:
+                setattr(self, k, v)
+
+    def has_attrib(self, attrib_name):
+        return attrib_name in self.__dict__
+
     def add(self, other):
         """
-        This let's you add on `other` genet.Network to the network this method is called on.
+        This lets you add on `other` genet.Network to the network this method is called on.
         This is deliberately not a magic function to discourage `new_network = network_1 + network_2` (and memory
         goes out the window)
         :param other:
@@ -243,10 +259,15 @@ def simplify(self, no_processes=1, keep_loops=False):
                                                 v not in useless_self_loops}
 
         # mark graph as having been simplified
-        self.graph.graph["simplified"] = True
+        self._mark_as_simplified()
+
+    def _mark_as_simplified(self):
+        self.attributes['simplified'] = {'name': 'simplified', 'class': 'java.lang.String', 'text': 'true'}
 
     def is_simplified(self):
-        return self.graph.graph["simplified"]
+        if 'simplified' in self.attributes:
+            return self.attributes['simplified']['text'] in {'true', 'True', True}
+        return False
 
     def node_attribute_summary(self, data=False):
         """

genet/inputs_handler/matsim_reader.py → genet/input/matsim_reader.py

@@ -10,7 +10,7 @@
 from genet.utils import spatial
 
 
-def read_node(elem, g, node_id_mapping, transformer):
+def read_node(elem, g, node_id_mapping, node_attribs, transformer):
     """
     Adds node elem of the stream to the network
     :param elem:
@@ -31,6 +31,10 @@ def read_node(elem, g, node_id_mapping, transformer):
     # lat and long values if so, but there is no obvious way to interrogate the transformer
     attribs['lon'], attribs['lat'] = lon, lat
     attribs['s2_id'] = spatial.generate_index_s2(lat, lon)
+
+    if node_attribs:
+        attribs['attributes'] = node_attribs
+
     node_id = attribs['id']
     if node_id in node_id_mapping:
         logging.warning('This MATSim network has a node that is not unique: {}. Generating a new id would'
@@ -99,13 +103,18 @@ def read_link(elem, g, u, v, node_id_mapping, link_id_mapping, link_attribs):
     return g, u, v, link_id_mapping, duplicated_link_id
 
 
-def read_link_attrib(elem, link_attribs):
+def update_additional_attrib(elem, attribs):
     """
-    Reads link attributes
+    Reads additional attributes
     :param elem:
-    :param link_attribs: current link attributes
+    :param attribs: current additional attributes
     :return:
     """
+    attribs[elem.attrib['name']] = read_additional_attrib(elem)
+    return attribs
+
+
+def read_additional_attrib(elem):
     d = elem.attrib
     if elem.text is None:
         d['text'] = ''
@@ -114,8 +123,7 @@ def read_link_attrib(elem, link_attribs):
         d['text'] = set(elem.text.split(','))
     else:
         d['text'] = elem.text
-    link_attribs[elem.attrib['name']] = d
-    return link_attribs
+    return d
 
 
 def unique_link_id(link_id, link_id_mapping):
@@ -145,40 +153,52 @@ def read_network(network_path, transformer: Transformer):
     """
     g = nx.MultiDiGraph()
 
+    network_attributes = {}
     node_id_mapping = {}
+    node_attribs = {}
     link_id_mapping = {}
     link_attribs = {}
     duplicated_link_ids = {}
     duplicated_node_ids = {}
     u, v = None, None
 
-    for event, elem in ET.iterparse(network_path):
-        if elem.tag == 'node':
-            g, duplicated_node_id = read_node(elem, g, node_id_mapping, transformer)
-            if duplicated_node_id:
-                for key, val in duplicated_node_id.items():
-                    if key in duplicated_node_ids:
-                        duplicated_node_ids[key].append(val)
-                    else:
-                        duplicated_node_ids[key] = [val]
-        elif elem.tag == 'attribute':
-            if node_id_mapping:
-                link_attribs = read_link_attrib(elem, link_attribs)
-            # else the attribute is on network level and does not belong to any nodes or links
-            elif elem.attrib['name'] == 'simplified':
-                g.graph['simplified'] = 'True' == elem.text
-        elif elem.tag == 'link':
-            g, u, v, link_id_mapping, duplicated_link_id = read_link(
-                elem, g, u, v, node_id_mapping, link_id_mapping, link_attribs)
-            if duplicated_link_id:
-                for key, val in duplicated_link_id.items():
-                    if key in duplicated_link_ids:
-                        duplicated_link_ids[key].append(val)
-                    else:
-                        duplicated_link_ids[key] = [val]
-            # reset link_attribs
-            link_attribs = {}
-    return g, link_id_mapping, duplicated_node_ids, duplicated_link_ids
+    elem_themes_for_additional_attributes = {'network', 'nodes', 'links'}
+    elem_type_for_additional_attributes = None
+
+    for event, elem in ET.iterparse(network_path, events=('start', 'end')):
+        if event == 'start':
+            if elem.tag in elem_themes_for_additional_attributes:
+                elem_type_for_additional_attributes = elem.tag
+        elif event == 'end':
+            if elem.tag == 'node':
+                g, duplicated_node_id = read_node(elem, g, node_id_mapping, node_attribs, transformer)
+                if duplicated_node_id:
+                    for key, val in duplicated_node_id.items():
+                        if key in duplicated_node_ids:
+                            duplicated_node_ids[key].append(val)
+                        else:
+                            duplicated_node_ids[key] = [val]
+                # reset node_attribs
+                node_attribs = {}
+            elif elem.tag == 'link':
+                g, u, v, link_id_mapping, duplicated_link_id = read_link(
+                    elem, g, u, v, node_id_mapping, link_id_mapping, link_attribs)
+                if duplicated_link_id:
+                    for key, val in duplicated_link_id.items():
+                        if key in duplicated_link_ids:
+                            duplicated_link_ids[key].append(val)
+                        else:
+                            duplicated_link_ids[key] = [val]
+                # reset link_attribs
+                link_attribs = {}
+            elif elem.tag == 'attribute':
+                if elem_type_for_additional_attributes == 'links':
+                    link_attribs = update_additional_attrib(elem, link_attribs)
+                elif elem_type_for_additional_attributes == 'network':
+                    network_attributes = update_additional_attrib(elem, network_attributes)
+                elif elem_type_for_additional_attributes == 'nodes':
+                    node_attribs = update_additional_attrib(elem, node_attribs)
+    return g, link_id_mapping, duplicated_node_ids, duplicated_link_ids, network_attributes
 
 
 def read_schedule(schedule_path, epsg):
@@ -193,7 +213,7 @@ def read_schedule(schedule_path, epsg):
 
     def write_transitLinesTransitRoute(transitLine, transitRoutes, transportMode):
         mode = transportMode['transportMode']
-        service_id = transitLine['transitLine']['id']
+        service_id = transitLine['transitLine_data']['id']
         service_routes = []
         for transitRoute, transitRoute_val in transitRoutes.items():
             stops = [Stop(
@@ -238,7 +258,7 @@ def write_transitLinesTransitRoute(transitLine, transitRoutes, transportMode):
                 trips['vehicle_id'].append(dep['departure']['vehicleRefId'])
 
             r = Route(
-                route_short_name=transitLine['transitLine']['name'],
+                route_short_name=transitLine['transitLine_data']['name'],
                 mode=mode,
                 stops=stops,
                 route=route,
@@ -248,8 +268,13 @@ def write_transitLinesTransitRoute(transitLine, transitRoutes, transportMode):
                 id=transitRoute,
                 await_departure=await_departure
             )
+            if transitRoute_val['attributes']:
+                r.add_additional_attributes({'attributes': transitRoute_val['attributes']})
             service_routes.append(r)
-        services.append(Service(id=service_id, routes=service_routes))
+        _service = Service(id=service_id, routes=service_routes)
+        if transitLine['attributes']:
+            _service.add_additional_attributes({'attributes': transitLine['attributes']})
+        services.append(_service)
 
     transitLine = {}
     transitRoutes = {}
@@ -258,64 +283,96 @@ def write_transitLinesTransitRoute(transitLine, transitRoutes, transportMode):
     is_minimalTransferTimes = False
     minimalTransferTimes = {}  # {'stop_id_1': {'stop_id_2': 0.0}} seconds_to_transfer between stop_id_1 and stop_id_2
 
+    elem_themes_for_additional_attributes = {'transitSchedule', 'stopFacility', 'transitLine', 'transitRoute'}
+    elem_type_for_additional_attributes = None
+    schedule_attribs = {}
+    # Track IDs through the stream
+    current_stop_id = None
+    current_route_id = None
+
     # transitLines
     for event, elem in ET.iterparse(schedule_path, events=('start', 'end')):
         if event == 'start':
+            if elem.tag in elem_themes_for_additional_attributes:
+                elem_type_for_additional_attributes = elem.tag
+
             if elem.tag == 'stopFacility':
                 attribs = elem.attrib
                 attribs['epsg'] = epsg
                 attribs['x'] = float(attribs['x'])
                 attribs['y'] = float(attribs['y'])
                 if attribs['id'] not in transit_stop_id_mapping:
                     transit_stop_id_mapping[attribs['id']] = attribs
+                current_stop_id = attribs['id']
 
-            if elem.tag == 'minimalTransferTimes':
+            elif elem.tag == 'minimalTransferTimes':
                 is_minimalTransferTimes = not is_minimalTransferTimes
-            if elem.tag == 'relation':
+            elif elem.tag == 'relation':
                 if is_minimalTransferTimes:
                     attribs = elem.attrib
                     minimalTransferTimes = dict_support.merge_complex_dictionaries(
                         minimalTransferTimes,
                         {attribs['fromStop']: {attribs['toStop']: float(attribs['transferTime'])}}
                     )
-            if elem.tag == 'transitLine':
+            elif elem.tag == 'transitLine':
                 if transitLine:
                     write_transitLinesTransitRoute(transitLine, transitRoutes, transportMode)
-                transitLine = {"transitLine": elem.attrib}
+                transitLine = {"transitLine_data": elem.attrib, 'attributes': {}}
                 transitRoutes = {}
 
-            if elem.tag == 'transitRoute':
+            elif elem.tag == 'transitRoute':
                 transitRoutes[elem.attrib['id']] = {'stops': [], 'links': [], 'departure_list': [],
-                                                    'attribs': elem.attrib}
-                transitRoute = elem.attrib['id']
+                                                    'attributes': {}}
+                current_route_id = elem.attrib['id']
 
             # doesn't have any attribs
             # if elem.tag == 'routeProfile':
             #     routeProfile = {'routeProfile': elem.attrib}
 
-            if elem.tag == 'stop':
-                transitRoutes[transitRoute]['stops'].append({'stop': elem.attrib})
+            elif elem.tag == 'stop':
+                transitRoutes[current_route_id]['stops'].append({'stop': elem.attrib})
 
             # doesn't have any attribs
             # if elem.tag == 'route':
             #     route = {'route': elem.attrib}
 
-            if elem.tag == 'link':
-                transitRoutes[transitRoute]['links'].append({'link': elem.attrib})
+            elif elem.tag == 'link':
+                transitRoutes[current_route_id]['links'].append({'link': elem.attrib})
 
             # doesn't have any attribs
             # if elem.tag == 'departures':
             #     departures = {'departures': elem.attrib}
 
-            if elem.tag == 'departure':
-                transitRoutes[transitRoute]['departure_list'].append({'departure': elem.attrib})
+            elif elem.tag == 'departure':
+                transitRoutes[current_route_id]['departure_list'].append({'departure': elem.attrib})
+            elif elem.tag == 'attribute':
+                if elem_type_for_additional_attributes == 'transitSchedule':
+                    schedule_attribs = update_additional_attrib(elem, schedule_attribs)
+                elif elem_type_for_additional_attributes == 'stopFacility':
+                    current_stop_data = transit_stop_id_mapping[current_stop_id]
+                    if 'attributes' in current_stop_data:
+                        current_stop_data['attributes'] = update_additional_attrib(
+                            elem,
+                            transit_stop_id_mapping[current_stop_id]['attributes'])
+                    else:
+                        current_stop_data['attributes'] = update_additional_attrib(elem, {})
+                elif elem_type_for_additional_attributes == 'transitLine':
+                    transitLine['attributes'] = update_additional_attrib(
+                        elem,
+                        transitLine['attributes']
+                    )
+                elif elem_type_for_additional_attributes == 'transitRoute':
+                    transitRoutes[current_route_id]['attributes'] = update_additional_attrib(
+                        elem,
+                        transitRoutes[current_route_id]['attributes']
+                    )
         elif (event == 'end') and (elem.tag == "transportMode"):
             transportMode = {'transportMode': elem.text}
 
     # add the last one
     write_transitLinesTransitRoute(transitLine, transitRoutes, transportMode)
 
-    return services, minimalTransferTimes, transit_stop_id_mapping
+    return services, minimalTransferTimes, transit_stop_id_mapping, schedule_attribs
 
 
 def read_vehicles(vehicles_path):

genet/inputs_handler/osm_reader.py → genet/input/osm_reader.py

@@ -4,10 +4,10 @@
 from pyproj import Transformer
 from math import ceil
 
-import genet.inputs_handler.osmnx_customised as osmnx_customised
+import genet.input.osmnx_customised as osmnx_customised
 import genet.utils.parallel as parallel
 import genet.utils.spatial as spatial
-from genet.outputs_handler.matsim_xml_values import MATSIM_JOSM_DEFAULTS
+from genet.output.matsim_xml_values import MATSIM_JOSM_DEFAULTS
 
 
 class Config(object):